1 int
2 __libc_system (const char *line)
3 {
4 if (line == NULL)
5 /* Check that we have a command processor available. It might
6 not be available after a chroot(), for example. */
7 return do_system ("exit 0") == 0;
8
9 return do_system (line);
10 }
11 weak_alias (__libc_system, system)
代码位于glibc/sysdeps/posix/system.c，这里system是__libc_system的弱别名，而__libc_system是do_system的前端函数，进行了参数的检查，接下来看do_system函数。
1 static intdo_system
2 do_system (const char *line)
3 {
4 int status, save;
5 pid_t pid;
6 struct sigaction sa;
7 #ifndef _LIBC_REENTRANT
8 struct sigaction intr, quit;
9 #endif
10 sigset_t omask;
11
12 sa.sa_handler = SIG_IGN;
13 sa.sa_flags = 0;
14 __sigemptyset (&sa.sa_mask);
15
16 DO_LOCK ();
17 if (ADD_REF () == 0)
18 {
19 if (__sigaction (SIGINT, &sa, &intr) < 0)
20 {
21 (void) SUB_REF ();
22 goto out;
23 }
24 if (__sigaction (SIGQUIT, &sa, &quit) < 0)
25 {
26 save = errno;
27 (void) SUB_REF ();
28 goto out_restore_sigint;
29 }
30 }
31 DO_UNLOCK ();
32
33 /* We reuse the bitmap in the 'sa' structure. */
34 __sigaddset (&sa.sa_mask, SIGCHLD);
35 save = errno;
36 if (__sigprocmask (SIG_BLOCK, &sa.sa_mask, &omask) < 0)
37 {
38 #ifndef _LIBC
39 if (errno == ENOSYS)
40 __set_errno (save);
41 else
42 #endif
43 {
44 DO_LOCK ();
45 if (SUB_REF () == 0)
46 {
47 save = errno;
48 (void) __sigaction (SIGQUIT, &quit, (struct sigaction *) NULL);
49 out_restore_sigint:
50 (void) __sigaction (SIGINT, &intr, (struct sigaction *) NULL);
51 __set_errno (save);
52 }
53 out:
54 DO_UNLOCK ();
55 return -1;
56 }
57 }
58
59 #ifdef CLEANUP_HANDLER
60 CLEANUP_HANDLER;
61 #endif
62
63 #ifdef FORK
64 pid = FORK ();
65 #else
66 pid = __fork ();
67 #endif
68 if (pid == (pid_t) 0)
69 {
70 /* Child side. */
71 const char *new_argv[4];
72 new_argv[0] = SHELL_NAME;
73 new_argv[1] = "-c";
74 new_argv[2] = line;
75 new_argv[3] = NULL;
76
77 /* Restore the signals. */
78 (void) __sigaction (SIGINT, &intr, (struct sigaction *) NULL);
79 (void) __sigaction (SIGQUIT, &quit, (struct sigaction *) NULL);
80 (void) __sigprocmask (SIG_SETMASK, &omask, (sigset_t *) NULL);
81 INIT_LOCK ();
82
83 /* Exec the shell. */
84 (void) __execve (SHELL_PATH, (char *const *) new_argv, __environ);
85 _exit (127);
86 }
87 else if (pid < (pid_t) 0)
88 /* The fork failed. */
89 status = -1;
90 else
91 /* Parent side. */
92 {
93 /* Note the system() is a cancellation point. But since we call
94 waitpid() which itself is a cancellation point we do not
95 have to do anything here. */
96 if (TEMP_FAILURE_RETRY (__waitpid (pid, &status, 0)) != pid)
97 status = -1;
98 }
99
100 #ifdef CLEANUP_HANDLER
101 CLEANUP_RESET;
102 #endif
103
104 save = errno;
105 DO_LOCK ();
106 if ((SUB_REF () == 0
107 && (__sigaction (SIGINT, &intr, (struct sigaction *) NULL)
108 | __sigaction (SIGQUIT, &quit, (struct sigaction *) NULL)) != 0)
109 || __sigprocmask (SIG_SETMASK, &omask, (sigset_t *) NULL) != 0)
110 {
111 #ifndef _LIBC
112 /* glibc cannot be used on systems without waitpid. */
113 if (errno == ENOSYS)
114 __set_errno (save);
115 else
116 #endif
117 status = -1;
118 }
119 DO_UNLOCK ();
120
121 return status;
122 }
首先函数设置了一些信号处理程序，来处理SIGINT和SIGQUIT信号，此处我们不过多关心，关键代码段在这里
1 #ifdef FORK
2 pid = FORK ();
3 #else
4 pid = __fork ();
5 #endif
6 if (pid == (pid_t) 0)
7 {
8 /* Child side. */
9 const char *new_argv[4];
10 new_argv[0] = SHELL_NAME;
11 new_argv[1] = "-c";
12 new_argv[2] = line;
13 new_argv[3] = NULL;
14
15 /* Restore the signals. */
16 (void) __sigaction (SIGINT, &intr, (struct sigaction *) NULL);
17 (void) __sigaction (SIGQUIT, &quit, (struct sigaction *) NULL);
18 (void) __sigprocmask (SIG_SETMASK, &omask, (sigset_t *) NULL);
19 INIT_LOCK ();
20
21 /* Exec the shell. */
22 (void) __execve (SHELL_PATH, (char *const *) new_argv, __environ);
23 _exit (127);
24 }
25 else if (pid < (pid_t) 0)
26 /* The fork failed. */
27 status = -1;
28 else
29 /* Parent side. */
30 {
31 /* Note the system() is a cancellation point. But since we call
32 waitpid() which itself is a cancellation point we do not
33 have to do anything here. */
34 if (TEMP_FAILURE_RETRY (__waitpid (pid, &status, 0)) != pid)
35 status = -1;
36 }
首先通过前端函数调用系统调用fork产生一个子进程，其中fork有两个返回值，对父进程返回子进程的pid，对子进程返回0。所以子进程执行6-24行代码，父进程执行30-35行代码。
子进程的逻辑非常清晰，调用execve执行SHELL_PATH指定的程序，参数通过new_argv传递，环境变量为全局变量__environ。
其中SHELL_PATH和SHELL_NAME定义如下
1 #define SHELL_PATH "/bin/sh" /* Path of the shell. */
2 #define SHELL_NAME "sh" /* Name to give it. */
其实就是生成一个子进程调用/bin/sh -c "命令"来执行向system传入的命令。
下面其实是我研究system函数的原因与重点：
在CTF的pwn题中，通过栈溢出调用system函数有时会失败，听师傅们说是环境变量被覆盖，但是一直都是懵懂，今天深入学习了一下，总算搞明白了。
在这里system函数需要的环境变量储存在全局变量__environ中，那么这个变量的内容是什么呢。
__environ是在glibc/csu/libc-start.c中定义的，我们来看几个关键语句。
# define LIBC_START_MAIN __libc_start_main
__libc_start_main是_start调用的函数，这涉及到程序开始时的一些初始化工作，对这些名词不了解的话可以看一下这篇文章。接下来看LIBC_START_MAIN函数。
1 STATIC int
2 LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
3 int argc, char **argv,
4 #ifdef LIBC_START_MAIN_AUXVEC_ARG
5 ElfW(auxv_t) *auxvec,
6 #endif
7 __typeof (main) init,
8 void (*fini) (void),
9 void (*rtld_fini) (void), void *stack_end)
10 {
11 /* Result of the 'main' function. */
12 int result;
13
14 __libc_multiple_libcs = &_dl_starting_up && !_dl_starting_up;
15
16 #ifndef SHARED
17 char **ev = &argv[argc + 1];
18
19 __environ = ev;
20
21 /* Store the lowest stack address. This is done in ld.so if this is
22 the code for the DSO. */
23 __libc_stack_end = stack_end;
......
202 /* Nothing fancy, just call the function. */
203 result = main (argc, argv, __environ MAIN_AUXVEC_PARAM);
204 #endif
205
206 exit (result);
207 }
我们可以看到，在没有define SHARED的情况下，在第19行定义了__environ的值。启动程序调用LIBC_START_MAIN之前，会先将环境变量和argv中的字符串保存起来（其实是保存到栈上），然后依次将环境变量中各项字符串的地址，argv中各项字符串的地址和argc入栈，所以环境变量数组一定位于argv数组的正后方，以一个空地址间隔。所以第17行的&argv[argc + 1]语句就是取环境变量数组在栈上的首地址，保存到ev中，最终保存到__environ中。第203行调用main函数，会将__environ的值入栈，这个被栈溢出覆盖掉没什么问题，只要保证__environ中的地址处不被覆盖即可。
所以，当栈溢出的长度过大，溢出的内容覆盖了__environ中地址中的重要内容时，调用system函数就会失败。具体环境变量距离溢出地址有多远，可以通过在_start中下断查看。