kdump+crash solves the crash problem and obtains global variables and malloc heap variables

Table of Contents

1. Experimental purpose

2. Experimental steps

3. Global variable analysis ideas

4. Obtain global variables inside the kernel

5. Modular driver global variables

5.1. p command prints modular global variable errors

5.2. sym/rd + struct command parses global variables in xxx.ko

5.3. Use the rd command to parse the global variables in xxx.ko

6. Obtaining heap variables

Environment: arm64, Linux version 5.10.66

1. Experimental purpose

Use crash to parse global variables KdumpStack_st stKdumpStack

The experimental program is as follows. When the program is compiled into ko and loaded into the device using insmod, enter the echo kdump-3 > /proc/dbug/dump command on the serial port and then execute our test program. After kdump generates the vmcore file, use the crash command to parse itContents of the global variable KdumpStack_st stKdumpStack.

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <asm/uaccess.h>
#include <asm/irq.h>
#include <asm/io.h>

#include <linux/device.h>
#include <linux/proc_fs.h>
#include <linux/version.h>

#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
/*
If the write is successful, the number of bytes written must be returned, otherwise Linux will still try to write
 */

#define PROC_DIR_NAME "dbug" //Folder name
#define PROC_FILE_NAME "kdump" //File name
#define KBUFSIZE 40


#define NAMELEN 20

typedef struct kdump_stack {
char cName[NAMELEN];
int iNumbers;
struct list_head list;
void (*printk)(void);
} KdumpStack_st;

typedef struct kdump_node {
char cName[NAMELEN];
int iNumber;
struct list_head list;
} KdumpNode_st;

typedef struct kdump_localval {
char cName[NAMELEN];
int iNumber;
} KdumpLkval_st;
static KdumpLkval_st *gpstKdLkval = NULL;
static KdumpStack_st stKdumpStack = {0};

void kdump_printk(void)
{
printk("%s %s %d", __FILE__, __func__, __LINE__);
/* Customize the way to trigger kdump */
panic("kdump test,Parse stack data");
}

int kdump_stack_test(int num, KdumpStack_st *pstKdStack, KdumpLkval_st *pstKdLkval)
{
int iCycle = 0;
char cNodeName[NAMELEN] = {0};
KdumpNode_st *pstKdNode = NULL;
/* View linked list */
for(iCycle = 0; iCycle < num; iCycle + + ) {
pstKdNode = kzalloc(sizeof(KdumpNode_st), GFP_KERNEL);
snprintf(cNodeName, NAMELEN, "node_%d", iCycle);
strncpy(pstKdNode->cName, cNodeName, NAMELEN);
pstKdNode->iNumber = iCycle;
\t\t
list_add_tail( & amp;pstKdNode->list, & amp;pstKdStack->list);
}
\t
/* Structure variables in the stack */
pstKdLkval->iNumber = 101;
/* Structure variables in the heap */
gpstKdLkval->iNumber = 201;
/* Global structure variables */
pstKdStack->printk();

return 0;
}

int kdump_proc_write(int num)
{

KdumpLkval_st stKdLkval = {0};
/* linked list */
INIT_LIST_HEAD( & amp;stKdumpStack.list);
/* Global structure variables */
strncpy(stKdumpStack.cName, "StTest", NAMELEN);
stKdumpStack.iNumbers = num;
stKdumpStack.printk = kdump_printk;
/* Structure variables in the stack */
strncpy(stKdLkval.cName, "local val", NAMELEN);
stKdLkval.iNumber = 100;
/* Structure variables in the heap */
gpstKdLkval = kzalloc(sizeof(KdumpLkval_st), GFP_KERNEL);
strncpy(gpstKdLkval->cName, "heap val", NAMELEN);
gpstKdLkval->iNumber = 200;
\t
kdump_stack_test(num, & amp;stKdumpStack, & amp;stKdLkval);
return 0;
}

/* The following is code unrelated to testing, it just creates a kdump debugging file in the /proc/dbug/ directory;
Enter the echo kdump-3 > /proc/dbug/dump command under the serial port and execute our test program */

char kbuf[KBUFSIZE] = {0}; //Save the data passed in from the user layer
struct proc_dir_entry *proc_wrbuff_dir;

static int proc_wrbuff_open(struct inode *inode,struct file *file);
static ssize_t proc_wrbuff_read(struct file *file, char __user *ubuf, size_t count, loff_t *offset);
static ssize_t proc_wrbuff_write(struct file *file, const char __user *ubuf, size_t count, loff_t *offset);


static int proc_wrbuff_open(struct inode *inode,struct file *file) {
printk("open embedsky board device!\
");
return 0;
}

static ssize_t proc_wrbuff_read(struct file *file, char __user *ubuf, size_t count, loff_t *offset) {
if (count > strlen(kbuf))
count = strlen(kbuf);
if (count < 0 )
return -3;
if (copy_to_user(ubuf, kbuf, count)) {
printk(KERN_ERR "copy_to_user failed! \
");
return -4;
}
return count;
}

static ssize_t proc_wrbuff_write(struct file *file, const char __user *ubuf, size_t count, loff_t *offset) {
\t
int num = 0;
size_t cnt = min((int)count, KBUFSIZE - 1);
if(copy_from_user(kbuf,ubuf,cnt)) {
printk(KERN_ERR "copy_to_user failed! \
");
return -EFAULT;
}
kbuf[cnt] = '\0';
printk("printk kbuf %s \
",kbuf);

if(sscanf(kbuf, "kdump-%d", & amp;num)) {
kdump_proc_write(num);
}

return cnt;
}

#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 10, 0)
static struct proc_ops fops_proc_wrbuffer = {

.proc_open = proc_wrbuff_open,
.proc_read = proc_wrbuff_read,
.proc_write = proc_wrbuff_write,
 };
#else
static struct file_operations fops_proc_wrbuffer = {
.owner = THIS_MODULE,
.open = proc_wrbuff_open,
.read = proc_wrbuff_read,
.write = proc_wrbuff_write,
.owner = THIS_MODULE,
 };
#endif


static int __init proc_wrbuff_init(void) {
int ret = 0;
struct proc_dir_entry *proc_file;

/* 1 create parent dir in /porc/dbug */
proc_wrbuff_dir = proc_mkdir(PROC_DIR_NAME, NULL);
if(!proc_wrbuff_dir){
printk("create proc dir error!");
return -1;
}
/* 2creata device file in /proc/parent dir*/
proc_file = proc_create_data(PROC_FILE_NAME, 0666, proc_wrbuff_dir, & amp;fops_proc_wrbuffer,0);
if (!proc_file) {
printk("create proc file error!");
ret = -2;
goto no_proc_file;
}

return 0;

no_proc_file:
remove_proc_entry(PROC_FILE_NAME,proc_wrbuff_dir);
return ret;
}

static void __exit proc_wrbuff_exit(void) {

remove_proc_entry(PROC_FILE_NAME,proc_wrbuff_dir);
remove_proc_entry(PROC_DIR_NAME, NULL);
}
late_initcall(proc_wrbuff_init);
module_exit(proc_wrbuff_exit);

MODULE_DESCRIPTION("debug");
MODULE_LICENSE("GPL");

2. Experimental steps

Prerequisite: The device Linux supports the kdump function. For support methods, please refer to this article.

a. Compile the test module: Compile the above source code into the kdump.ko driver module, and use the insmod command in the device serial port to load this driver into the device. At this time, the /proc/dbug/kdump debug file will be generated

b. Trigger panic: Execute the echo kdump-4 > /proc/dbug/kdump command to execute the test program. The test program will execute panic to trigger the kdump mechanism, start the capture kernel, and generate the /proc/vmcore file in the capture kernel.

c. Save the vmcore file: execute cd /proc;tar -czf /tmp/3588/vmcore.tar.gz ./vmcore to compress the vmcore file under the captured kernel and copy it to a USB disk or NFS mounted directory.

d. Use crash to analyze the vmcore file: Execute the crash vmlinux vmcore command to use crash to analyze the vmcore file.

e. Since the vmcore file will only retain the code part of kdump.ko, you need to use mod to load the debugging and symbol information of the kdump.ko driver module in crash. Executing the crash> dis -l kdump_proc_write command in this way will correctly display the line number information corresponding to the assembly code.

mod -s kdump /kdump/demo/stack/kdump.ko

3. Global variable analysis ideas

Global variables are stored in the static data segment, and their life cycle is from the beginning of the program to the end of the program, in the global scope; therefore, the crash tool has a special command p to read the contents of global variables; but actual testing found that p can only read limited global variables. Global variables defined inside the kernel, such as process 0: struct task_struct init_task.

For modular drivers, such as global variables defined in xxx.ko, the p command in crash cannot correctly parse the global variables in xxx.ko (although mod -s xxx xxx.ko is used to load them in crash Driver debugging information); in actual measurement, you can use the sys/rd + struct command combination to obtain the contents of the global structure.

4. Get global variables inside kernel

The p command in crash is used to print the value of the specified symbol. You can use help p in crash to view specific help information. A brief explanation is as follows:

SYNOPSIS
p [-x|-d][-u] [symbol[:cpuspec]]
-x hexadecimal display
-d decimal display
symbol displays the value of a global variable
symbol:x prints the value of a specific cpu for the global variable of percpu

Usage examples are as follows:

/* Print global variables in the kernel */
crash> p jiffies
jiffies = $1 = 4294947275
crash> p jiffies -x
jiffies = $2 = 0xffffb1cb
crash> p jiffies -d
jiffies = $3 = 4294947275

/* Print global variables of percpu */
crash> p irq_stat
PER-CPU DATA TYPE:
  irq_cpustat_t irq_stat;
PER-CPU ADDRESSES:
  [0]: ffffff81fced2980
  [1]: ffffff81fcef0980
  [2]: ffffff81fcf0e980
  [3]: ffffff81fcf2c980
  [4]: ffffff81fcf4a980
  [5]: ffffff81fcf68980
  [6]: ffffff81fcf86980
  [7]: ffffff81fcfa4980
  
/* Print the percpu variable of a certain CPU */
crash> p irq_stat:1
per_cpu(irq_stat, 1) = $4 = {
  __softirq_pending = 0
}

5. Modular driver global variables

5.1, p command prints modular global variable errors

An error occurs when using the p command for modular drivers.

/* Actual cName="StTest" iNumbers=4 */
crash> p stKdumpStack
stKdumpStack = $7 = {
  cName = "\001\005\200R\000\000\000\260\000P\002\221c\345\375\225\000\000", <incomplete sequence \324>,
  iNumbers = -1463714819,
  list = {
    next = 0xd65f03c0d50323bf,
    prev = 0xb9402c64d5384103
  },
  printk = 0xf9400462d503233f
}

5.2. sym/rd + struct command parses global variables in xxx.ko

a. sym/rd gets the address of the global structure

Use sym/rd to get the address of the global structure

crash> rd stKdumpStack
ffffffc008f0d4b0: 0000747365547453 StTest..
crash> sym stKdumpStack
ffffffc008f0d4b0 (b) stKdumpStack [kdump]
crash>

You can also use the rd command here to print out the contents of global variables.

b. struct gets the global structure content

Use struct to get global structure contents

crash> struct KdumpStack_st ffffffc008f0d4b0
struct KdumpStack_st {
cName = “StTest\000\000\000\000\000\000\000\000\000\000\000\000\000”,
iNumbers = 4,
list = {
next = 0xffffff8118002798,
prev = 0xffffff8118002618
},
printk = 0xffffffc008f0b4f4
}
crash>

5.3. Use the rd command to parse the global variables in xxx.ko

a. Use struct KdumpStack_st -o to confirm the structure size and the offset address of each member

crash> struct KdumpStack_st -o
typedef struct kdump_stack {
[0] char cName[20];
[20] int iNumbers;
[24] struct list_head list;
[40] void (*printk)(void);
} KdumpStack_st;
SIZE: 48

b. Use rd stKdumpStack 6 to read out the 48-byte content

crash> rd stKdumpStack 6
ffffffc008f0d4b0: 0000747365547453 0000000000000000 StTest…..
ffffffc008f0d4c0: 0000000400000000 ffffff8118002798 ………’……
ffffffc008f0d4d0: ffffff8118002618 ffffffc008f0b4f4 . & amp;…………
crash>

6. Acquisition of heap variables

Use kzalloc to allocate heap memory space. The address of the heap memory is generally saved using global variables; the key to obtaining the content of the variable is to obtain the pointer of the heap variable. Take the kdump_proc_write function as an example to obtain the heap content pointed to by the pointer gpstKdLkval.

int kdump_proc_write(int num)
{........................................................ .......
/* Structure variables in the heap */
gpstKdLkval = kzalloc(sizeof(KdumpLkval_st), GFP_KERNEL);
strncpy(gpstKdLkval->cName, "heap val", NAMELEN);
gpstKdLkval->iNumber = 200;
................................................................. .......
return 0;
}

a. Use rd to obtain the contents of the pointer KdumpLkval_st *gpstKdLkval

crash> rd gpstKdLkval
ffffffc008f0d4a8: ffffff8118002900…

b. Use struct to obtain the contents of heap variables

ffffff8118002900 is the heap memory address pointed by the pointer. Use the struct command to print out the heap content.

crash> struct KdumpLkval_st ffffff8118002900
struct KdumpLkval_st {
cName = “heap val\000\000\000\000\000\000\000\000\000\000\000”,
iNumber = 201
}
crash>

The knowledge points of the article match the official knowledge files, and you can further learn relevant knowledge. CS entry skill treeLinux introductionFirst introduction to Linux 37209 people are learning the system