aof是redis提供的一种数据持久化机制，通过将每一条命令dump下来，保持数据和内存中的数据一致。
1 #include "redis.h"
2 #include "bio.h"
3
4 #include <signal.h>
5 #include <fcntl.h>
6 #include <sys/stat.h>
7 #include <sys/types.h>
8 #include <sys/time.h>
9 #include <sys/resource.h>
10 #include <sys/wait.h>
11
12 void aofUpdateCurrentSize(void);
13
14 void aof_background_fsync(int fd) {
15 bioCreateBackgroundJob(REDIS_BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL);
16 }
17
18 /* Called when the user switches from "appendonly yes" to "appendonly no"
19 * at runtime using the CONFIG command. */
20 void stopAppendOnly(void) {
21 flushAppendOnlyFile(1);
22 aof_fsync(server.appendfd);
23 close(server.appendfd);
24
25 server.appendfd = -1;
26 server.appendseldb = -1;
27 server.appendonly = 0;
28 /* rewrite operation in progress? kill it, wait child exit */
29 if (server.bgrewritechildpid != -1) {
30 int statloc;
31
32 if (kill(server.bgrewritechildpid,SIGKILL) != -1)
33 wait3(&statloc,0,NULL);
34 /* reset the buffer accumulating changes while the child saves */
35 sdsfree(server.bgrewritebuf);
36 server.bgrewritebuf = sdsempty();
37 server.bgrewritechildpid = -1;
38 }
39 }
40
41 /* Called when the user switches from "appendonly no" to "appendonly yes"
42 * at runtime using the CONFIG command. */
43 int startAppendOnly(void) {
44 server.appendonly = 1;
45 server.lastfsync = time(NULL);
46 server.appendfd = open(server.appendfilename,O_WRONLY|O_APPEND|O_CREAT,0644);
47 if (server.appendfd == -1) {
48 redisLog(REDIS_WARNING,"Used tried to switch on AOF via CONFIG, but I can't open the AOF file: %s",strerror(errno));
49 return REDIS_ERR;
50 }
51 if (rewriteAppendOnlyFileBackground() == REDIS_ERR) {
52 server.appendonly = 0;
53 close(server.appendfd);
54 redisLog(REDIS_WARNING,"Used tried to switch on AOF via CONFIG, I can't trigger a background AOF rewrite operation. Check the above logs for more info about the error.",strerror(errno));
55 return REDIS_ERR;
56 }
57 return REDIS_OK;
58 }
59
60 /* Write the append only file buffer on disk.
61 *
62 * Since we are required to write the AOF before replying to the client,
63 * and the only way the client socket can get a write is entering when the
64 * the event loop, we accumulate all the AOF writes in a memory
65 * buffer and write it on disk using this function just before entering
66 * the event loop again.
67 *
68 * About the 'force' argument:
69 *
70 * When the fsync policy is set to 'everysec' we may delay the flush if there
71 * is still an fsync() going on in the background thread, since for instance
72 * on Linux write(2) will be blocked by the background fsync anyway.
73 * When this happens we remember that there is some aof buffer to be
74 * flushed ASAP, and will try to do that in the serverCron() function.
75 *
76 * However if force is set to 1 we'll write regardless of the background
77 * fsync. */
78 void flushAppendOnlyFile(int force) {
79 ssize_t nwritten;
80 int sync_in_progress = 0;
81
82 if (sdslen(server.aofbuf) == 0) return;
83
84 if (server.appendfsync == APPENDFSYNC_EVERYSEC)
85 sync_in_progress = bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC) != 0;
86
87 if (server.appendfsync == APPENDFSYNC_EVERYSEC && !force) {
88 /* With this append fsync policy we do background fsyncing.
89 * If the fsync is still in progress we can try to delay
90 * the write for a couple of seconds. */
91 if (sync_in_progress) {
92 if (server.aof_flush_postponed_start == 0) {
93 /* No previous write postponinig, remember that we are
94 * postponing the flush and return. */
95 server.aof_flush_postponed_start = server.unixtime;
96 return;
97 } else if (server.unixtime - server.aof_flush_postponed_start < 2) {
98 /* We were already waiting for fsync to finish, but for less
99 * than two seconds this is still ok. Postpone again. */
100 return;
101 }
102 /* Otherwise fall trough, and go write since we can't wait
103 * over two seconds. */
104 redisLog(REDIS_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis.");
105 }
106 }
107 /* If you are following this code path, then we are going to write so
108 * set reset the postponed flush sentinel to zero. */
109 server.aof_flush_postponed_start = 0;
110
111 /* We want to perform a single write. This should be guaranteed atomic
112 * at least if the filesystem we are writing is a real physical one.
113 * While this will save us against the server being killed I don't think
114 * there is much to do about the whole server stopping for power problems
115 * or alike */
116 nwritten = write(server.appendfd,server.aofbuf,sdslen(server.aofbuf));
117 if (nwritten != (signed)sdslen(server.aofbuf)) {
118 /* Ooops, we are in troubles. The best thing to do for now is
119 * aborting instead of giving the illusion that everything is
120 * working as expected. */
121 if (nwritten == -1) {
122 redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno));
123 } else {
124 redisLog(REDIS_WARNING,"Exiting on short write while writing to the append-only file: %s",strerror(errno));
125 }
126 exit(1);
127 }
128 server.appendonly_current_size += nwritten;
129
130 /* Re-use AOF buffer when it is small enough. The maximum comes from the
131 * arena size of 4k minus some overhead (but is otherwise arbitrary). */
132 if ((sdslen(server.aofbuf)+sdsavail(server.aofbuf)) < 4000) {
133 sdsclear(server.aofbuf);
134 } else {
135 sdsfree(server.aofbuf);
136 server.aofbuf = sdsempty();
137 }
138
139 /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
140 * children doing I/O in the background. */
141 if (server.no_appendfsync_on_rewrite &&
142 (server.bgrewritechildpid != -1 || server.bgsavechildpid != -1))
143 return;
144
145 /* Perform the fsync if needed. */
146 if (server.appendfsync == APPENDFSYNC_ALWAYS) {
147 /* aof_fsync is defined as fdatasync() for Linux in order to avoid
148 * flushing metadata. */
149 aof_fsync(server.appendfd); /* Let's try to get this data on the disk */
150 server.lastfsync = server.unixtime;
151 } else if ((server.appendfsync == APPENDFSYNC_EVERYSEC &&
152 server.unixtime > server.lastfsync)) {
153 if (!sync_in_progress) aof_background_fsync(server.appendfd);
154 server.lastfsync = server.unixtime;
155 }
156 }
157
158 sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) {
159 char buf[32];
160 int len, j;
161 robj *o;
162
163 buf[0] = '*';
164 len = 1+ll2string(buf+1,sizeof(buf)-1,argc);
165 buf[len++] = '\r';
166 buf[len++] = '\n';
167 dst = sdscatlen(dst,buf,len);
168
169 for (j = 0; j < argc; j++) {
170 o = getDecodedObject(argv[j]);
171 buf[0] = '$';
172 len = 1+ll2string(buf+1,sizeof(buf)-1,sdslen(o->ptr));
173 buf[len++] = '\r';
174 buf[len++] = '\n';
175 dst = sdscatlen(dst,buf,len);
176 dst = sdscatlen(dst,o->ptr,sdslen(o->ptr));
177 dst = sdscatlen(dst,"\r\n",2);
178 decrRefCount(o);
179 }
180 return dst;
181 }
182
183 sds catAppendOnlyExpireAtCommand(sds buf, robj *key, robj *seconds) {
184 int argc = 3;
185 long when;
186 robj *argv[3];
187
188 /* Make sure we can use strtol */
189 seconds = getDecodedObject(seconds);
190 when = time(NULL)+strtol(seconds->ptr,NULL,10);
191 decrRefCount(seconds);
192
193 argv[0] = createStringObject("EXPIREAT",8);
194 argv[1] = key;
195 argv[2] = createObject(REDIS_STRING,
196 sdscatprintf(sdsempty(),"%ld",when));
197 buf = catAppendOnlyGenericCommand(buf, argc, argv);
198 decrRefCount(argv[0]);
199 decrRefCount(argv[2]);
200 return buf;
201 }
202
203 void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
204 sds buf = sdsempty();
205 robj *tmpargv[3];
206
207 /* The DB this command was targetting is not the same as the last command
208 * we appendend. To issue a SELECT command is needed. */
209 if (dictid != server.appendseldb) {
210 char seldb[64];
211
212 snprintf(seldb,sizeof(seldb),"%d",dictid);
213 buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
214 (unsigned long)strlen(seldb),seldb);
215 server.appendseldb = dictid;
216 }
217
218 if (cmd->proc == expireCommand) {
219 /* Translate EXPIRE into EXPIREAT */
220 buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
221 } else if (cmd->proc == setexCommand) {
222 /* Translate SETEX to SET and EXPIREAT */
223 tmpargv[0] = createStringObject("SET",3);
224 tmpargv[1] = argv[1];
225 tmpargv[2] = argv[3];
226 buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
227 decrRefCount(tmpargv[0]);
228 buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
229 } else {
230 buf = catAppendOnlyGenericCommand(buf,argc,argv);
231 }
232
233 /* Append to the AOF buffer. This will be flushed on disk just before
234 * of re-entering the event loop, so before the client will get a
235 * positive reply about the operation performed. */
236 server.aofbuf = sdscatlen(server.aofbuf,buf,sdslen(buf));
237
238 /* If a background append only file rewriting is in progress we want to
239 * accumulate the differences between the child DB and the current one
240 * in a buffer, so that when the child process will do its work we
241 * can append the differences to the new append only file. */
242 if (server.bgrewritechildpid != -1)
243 server.bgrewritebuf = sdscatlen(server.bgrewritebuf,buf,sdslen(buf));
244
245 sdsfree(buf);
246 }
247
248 /* In Redis commands are always executed in the context of a client, so in
249 * order to load the append only file we need to create a fake client. */
250 struct redisClient *createFakeClient(void) {
251 struct redisClient *c = zmalloc(sizeof(*c));
252
253 selectDb(c,0);
254 c->fd = -1;
255 c->querybuf = sdsempty();
256 c->argc = 0;
257 c->argv = NULL;
258 c->bufpos = 0;
259 c->flags = 0;
260 /* We set the fake client as a slave waiting for the synchronization
261 * so that Redis will not try to send replies to this client. */
262 c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
263 c->reply = listCreate();
264 c->reply_bytes = 0;
265 c->watched_keys = listCreate();
266 listSetFreeMethod(c->reply,decrRefCount);
267 listSetDupMethod(c->reply,dupClientReplyValue);
268 initClientMultiState(c);
269 return c;
270 }
271
272 void freeFakeClient(struct redisClient *c) {
273 sdsfree(c->querybuf);
274 listRelease(c->reply);
275 listRelease(c->watched_keys);
276 freeClientMultiState(c);
277 zfree(c);
278 }
279
280 /* Replay the append log file. On error REDIS_OK is returned. On non fatal
281 * error (the append only file is zero-length) REDIS_ERR is returned. On
282 * fatal error an error message is logged and the program exists. */
283 int loadAppendOnlyFile(char *filename) {
284 struct redisClient *fakeClient;
285 FILE *fp = fopen(filename,"r");
286 struct redis_stat sb;
287 int appendonly = server.appendonly;
288 long loops = 0;
289
290 if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
291 server.appendonly_current_size = 0;
292 fclose(fp);
293 return REDIS_ERR;
294 }
295
296 if (fp == NULL) {
297 redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
298 exit(1);
299 }
300
301 /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
302 * to the same file we're about to read. */
303 server.appendonly = 0;
304
305 fakeClient = createFakeClient();
306 startLoading(fp);
307
308 while(1) {
309 int argc, j;
310 unsigned long len;
311 robj **argv;
312 char buf[128];
313 sds argsds;
314 struct redisCommand *cmd;
315 int force_swapout;
316
317 /* Serve the clients from time to time */
318 if (!(loops++ % 1000)) {
319 loadingProgress(ftello(fp));
320 aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
321 }
322
323 if (fgets(buf,sizeof(buf),fp) == NULL) {
324 if (feof(fp))
325 break;
326 else
327 goto readerr;
328 }
329 if (buf[0] != '*') goto fmterr;
330 argc = atoi(buf+1);
331 if (argc < 1) goto fmterr;
332
333 argv = zmalloc(sizeof(robj*)*argc);
334 for (j = 0; j < argc; j++) {
335 if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr;
336 if (buf[0] != '$') goto fmterr;
337 len = strtol(buf+1,NULL,10);
338 argsds = sdsnewlen(NULL,len);
339 if (len && fread(argsds,len,1,fp) == 0) goto fmterr;
340 argv[j] = createObject(REDIS_STRING,argsds);
341 if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */
342 }
343
344 /* Command lookup */
345 cmd = lookupCommand(argv[0]->ptr);
346 if (!cmd) {
347 redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr);
348 exit(1);
349 }
350 /* Run the command in the context of a fake client */
351 fakeClient->argc = argc;
352 fakeClient->argv = argv;
353 cmd->proc(fakeClient);
354
355 /* The fake client should not have a reply */
356 redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
357 /* The fake client should never get blocked */
358 redisAssert((fakeClient->flags & REDIS_BLOCKED) == 0);
359
360 /* Clean up. Command code may have changed argv/argc so we use the
361 * argv/argc of the client instead of the local variables. */
362 for (j = 0; j < fakeClient->argc; j++)
363 decrRefCount(fakeClient->argv[j]);
364 zfree(fakeClient->argv);
365
366 /* Handle swapping while loading big datasets when VM is on */
367 force_swapout = 0;
368 if ((zmalloc_used_memory() - server.vm_max_memory) > 1024*1024*32)
369 force_swapout = 1;
370
371 if (server.vm_enabled && force_swapout) {
372 while (zmalloc_used_memory() > server.vm_max_memory) {
373 if (vmSwapOneObjectBlocking() == REDIS_ERR) break;
374 }
375 }
376 }
377
378 /* This point can only be reached when EOF is reached without errors.
379 * If the client is in the middle of a MULTI/EXEC, log error and quit. */
380 if (fakeClient->flags & REDIS_MULTI) goto readerr;
381
382 fclose(fp);
383 freeFakeClient(fakeClient);
384 server.appendonly = appendonly;
385 stopLoading();
386 aofUpdateCurrentSize();
387 server.auto_aofrewrite_base_size = server.appendonly_current_size;
388 return REDIS_OK;
389
390 readerr:
391 if (feof(fp)) {
392 redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file");
393 } else {
394 redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
395 }
396 exit(1);
397 fmterr:
398 redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
399 exit(1);
400 }
401
402 /* Write a sequence of commands able to fully rebuild the dataset into
403 * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
404 int rewriteAppendOnlyFile(char *filename) {
405 dictIterator *di = NULL;
406 dictEntry *de;
407 FILE *fp;
408 char tmpfile[256];
409 int j;
410 time_t now = time(NULL);
411
412 /* Note that we have to use a different temp name here compared to the
413 * one used by rewriteAppendOnlyFileBackground() function. */
414 snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid());
415 fp = fopen(tmpfile,"w");
416 if (!fp) {
417 redisLog(REDIS_WARNING, "Failed rewriting the append only file: %s", strerror(errno));
418 return REDIS_ERR;
419 }
420 for (j = 0; j < server.dbnum; j++) {
421 char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
422 redisDb *db = server.db+j;
423 dict *d = db->dict;
424 if (dictSize(d) == 0) continue;
425 di = dictGetSafeIterator(d);
426 if (!di) {
427 fclose(fp);
428 return REDIS_ERR;
429 }
430
431 /* SELECT the new DB */
432 if (fwrite(selectcmd,sizeof(selectcmd)-1,1,fp) == 0) goto werr;
433 if (fwriteBulkLongLong(fp,j) == 0) goto werr;
434
435 /* Iterate this DB writing every entry */
436 while((de = dictNext(di)) != NULL) {
437 sds keystr = dictGetEntryKey(de);
438 robj key, *o;
439 time_t expiretime;
440 int swapped;
441
442 keystr = dictGetEntryKey(de);
443 o = dictGetEntryVal(de);
444 initStaticStringObject(key,keystr);
445 /* If the value for this key is swapped, load a preview in memory.
446 * We use a "swapped" flag to remember if we need to free the
447 * value object instead to just increment the ref count anyway
448 * in order to avoid copy-on-write of pages if we are forked() */
449 if (!server.vm_enabled || o->storage == REDIS_VM_MEMORY ||
450 o->storage == REDIS_VM_SWAPPING) {
451 swapped = 0;
452 } else {
453 o = vmPreviewObject(o);
454 swapped = 1;
455 }
456 expiretime = getExpire(db,&key);
457
458 /* Save the key and associated value */
459 if (o->type == REDIS_STRING) {
460 /* Emit a SET command */
461 char cmd[]="*3\r\n$3\r\nSET\r\n";
462 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
463 /* Key and value */
464 if (fwriteBulkObject(fp,&key) == 0) goto werr;
465 if (fwriteBulkObject(fp,o) == 0) goto werr;
466 } else if (o->type == REDIS_LIST) {
467 /* Emit the RPUSHes needed to rebuild the list */
468 char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
469 if (o->encoding == REDIS_ENCODING_ZIPLIST) {
470 unsigned char *zl = o->ptr;
471 unsigned char *p = ziplistIndex(zl,0);
472 unsigned char *vstr;
473 unsigned int vlen;
474 long long vlong;
475
476 while(ziplistGet(p,&vstr,&vlen,&vlong)) {
477 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
478 if (fwriteBulkObject(fp,&key) == 0) goto werr;
479 if (vstr) {
480 if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
481 goto werr;
482 } else {
483 if (fwriteBulkLongLong(fp,vlong) == 0)
484 goto werr;
485 }
486 p = ziplistNext(zl,p);
487 }
488 } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
489 list *list = o->ptr;
490 listNode *ln;
491 listIter li;
492
493 listRewind(list,&li);
494 while((ln = listNext(&li))) {
495 robj *eleobj = listNodeValue(ln);
496
497 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
498 if (fwriteBulkObject(fp,&key) == 0) goto werr;
499 if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
500 }
501 } else {
502 redisPanic("Unknown list encoding");
503 }
504 } else if (o->type == REDIS_SET) {
505 char cmd[]="*3\r\n$4\r\nSADD\r\n";
506
507 /* Emit the SADDs needed to rebuild the set */
508 if (o->encoding == REDIS_ENCODING_INTSET) {
509 int ii = 0;
510 int64_t llval;
511 while(intsetGet(o->ptr,ii++,&llval)) {
512 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
513 if (fwriteBulkObject(fp,&key) == 0) goto werr;
514 if (fwriteBulkLongLong(fp,llval) == 0) goto werr;
515 }
516 } else if (o->encoding == REDIS_ENCODING_HT) {
517 dictIterator *di = dictGetIterator(o->ptr);
518 dictEntry *de;
519 while((de = dictNext(di)) != NULL) {
520 robj *eleobj = dictGetEntryKey(de);
521 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
522 if (fwriteBulkObject(fp,&key) == 0) goto werr;
523 if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
524 }
525 dictReleaseIterator(di);
526 } else {
527 redisPanic("Unknown set encoding");
528 }
529 } else if (o->type == REDIS_ZSET) {
530 /* Emit the ZADDs needed to rebuild the sorted set */
531 char cmd[]="*4\r\n$4\r\nZADD\r\n";
532
533 if (o->encoding == REDIS_ENCODING_ZIPLIST) {
534 unsigned char *zl = o->ptr;
535 unsigned char *eptr, *sptr;
536 unsigned char *vstr;
537 unsigned int vlen;
538 long long vll;
539 double score;
540
541 eptr = ziplistIndex(zl,0);
542 redisAssert(eptr != NULL);
543 sptr = ziplistNext(zl,eptr);
544 redisAssert(sptr != NULL);
545
546 while (eptr != NULL) {
547 redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
548 score = zzlGetScore(sptr);
549
550 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
551 if (fwriteBulkObject(fp,&key) == 0) goto werr;
552 if (fwriteBulkDouble(fp,score) == 0) goto werr;
553 if (vstr != NULL) {
554 if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
555 goto werr;
556 } else {
557 if (fwriteBulkLongLong(fp,vll) == 0)
558 goto werr;
559 }
560 zzlNext(zl,&eptr,&sptr);
561 }
562 } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
563 zset *zs = o->ptr;
564 dictIterator *di = dictGetIterator(zs->dict);
565 dictEntry *de;
566
567 while((de = dictNext(di)) != NULL) {
568 robj *eleobj = dictGetEntryKey(de);
569 double *score = dictGetEntryVal(de);
570
571 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
572 if (fwriteBulkObject(fp,&key) == 0) goto werr;
573 if (fwriteBulkDouble(fp,*score) == 0) goto werr;
574 if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
575 }
576 dictReleaseIterator(di);
577 } else {
578 redisPanic("Unknown sorted set encoding");
579 }
580 } else if (o->type == REDIS_HASH) {
581 char cmd[]="*4\r\n$4\r\nHSET\r\n";
582
583 /* Emit the HSETs needed to rebuild the hash */
584 if (o->encoding == REDIS_ENCODING_ZIPMAP) {
585 unsigned char *p = zipmapRewind(o->ptr);
586 unsigned char *field, *val;
587 unsigned int flen, vlen;
588
589 while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
590 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
591 if (fwriteBulkObject(fp,&key) == 0) goto werr;
592 if (fwriteBulkString(fp,(char*)field,flen) == 0)
593 goto werr;
594 if (fwriteBulkString(fp,(char*)val,vlen) == 0)
595 goto werr;
596 }
597 } else {
598 dictIterator *di = dictGetIterator(o->ptr);
599 dictEntry *de;
600
601 while((de = dictNext(di)) != NULL) {
602 robj *field = dictGetEntryKey(de);
603 robj *val = dictGetEntryVal(de);
604
605 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
606 if (fwriteBulkObject(fp,&key) == 0) goto werr;
607 if (fwriteBulkObject(fp,field) == 0) goto werr;
608 if (fwriteBulkObject(fp,val) == 0) goto werr;
609 }
610 dictReleaseIterator(di);
611 }
612 } else {
613 redisPanic("Unknown object type");
614 }
615 /* Save the expire time */
616 if (expiretime != -1) {
617 char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n";
618 /* If this key is already expired skip it */
619 if (expiretime < now) continue;
620 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
621 if (fwriteBulkObject(fp,&key) == 0) goto werr;
622 if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
623 }
624 if (swapped) decrRefCount(o);
625 }
626 dictReleaseIterator(di);
627 }
628
629 /* Make sure data will not remain on the OS's output buffers */
630 fflush(fp);
631 aof_fsync(fileno(fp));
632 fclose(fp);
633
634 /* Use RENAME to make sure the DB file is changed atomically only
635 * if the generate DB file is ok. */
636 if (rename(tmpfile,filename) == -1) {
637 redisLog(REDIS_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno));
638 unlink(tmpfile);
639 return REDIS_ERR;
640 }
641 redisLog(REDIS_NOTICE,"SYNC append only file rewrite performed");
642 return REDIS_OK;
643
644 werr:
645 fclose(fp);
646 unlink(tmpfile);
647 redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
648 if (di) dictReleaseIterator(di);
649 return REDIS_ERR;
650 }
651
652 /* This is how rewriting of the append only file in background works:
653 *
654 * 1) The user calls BGREWRITEAOF
655 * 2) Redis calls this function, that forks():
656 * 2a) the child rewrite the append only file in a temp file.
657 * 2b) the parent accumulates differences in server.bgrewritebuf.
658 * 3) When the child finished '2a' exists.
659 * 4) The parent will trap the exit code, if it's OK, will append the
660 * data accumulated into server.bgrewritebuf into the temp file, and
661 * finally will rename(2) the temp file in the actual file name.
662 * The the new file is reopened as the new append only file. Profit!
663 */
664 int rewriteAppendOnlyFileBackground(void) {
665 pid_t childpid;
666 long long start;
667
668 if (server.bgrewritechildpid != -1) return REDIS_ERR;
669 if (server.vm_enabled) waitEmptyIOJobsQueue();
670 start = ustime();
671 if ((childpid = fork()) == 0) {
672 char tmpfile[256];
673
674 /* Child */
675 if (server.vm_enabled) vmReopenSwapFile();
676 if (server.ipfd > 0) close(server.ipfd);
677 if (server.sofd > 0) close(server.sofd);
678 snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
679 if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
680 _exit(0);
681 } else {
682 _exit(1);
683 }
684 } else {
685 /* Parent */
686 server.stat_fork_time = ustime()-start;
687 if (childpid == -1) {
688 redisLog(REDIS_WARNING,
689 "Can't rewrite append only file in background: fork: %s",
690 strerror(errno));
691 return REDIS_ERR;
692 }
693 redisLog(REDIS_NOTICE,
694 "Background append only file rewriting started by pid %d",childpid);
695 server.aofrewrite_scheduled = 0;
696 server.bgrewritechildpid = childpid;
697 updateDictResizePolicy();
698 /* We set appendseldb to -1 in order to force the next call to the
699 * feedAppendOnlyFile() to issue a SELECT command, so the differences
700 * accumulated by the parent into server.bgrewritebuf will start
701 * with a SELECT statement and it will be safe to merge. */
702 server.appendseldb = -1;
703 return REDIS_OK;
704 }
705 return REDIS_OK; /* unreached */
706 }
707
708 void bgrewriteaofCommand(redisClient *c) {
709 if (server.bgrewritechildpid != -1) {
710 addReplyError(c,"Background append only file rewriting already in progress");
711 } else if (server.bgsavechildpid != -1) {
712 server.aofrewrite_scheduled = 1;
713 addReplyStatus(c,"Background append only file rewriting scheduled");
714 } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
715 addReplyStatus(c,"Background append only file rewriting started");
716 } else {
717 addReply(c,shared.err);
718 }
719 }
720
721 void aofRemoveTempFile(pid_t childpid) {
722 char tmpfile[256];
723
724 snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid);
725 unlink(tmpfile);
726 }
727
728 /* Update the server.appendonly_current_size filed explicitly using stat(2)
729 * to check the size of the file. This is useful after a rewrite or after
730 * a restart, normally the size is updated just adding the write length
731 * to the current lenght, that is much faster. */
732 void aofUpdateCurrentSize(void) {
733 struct redis_stat sb;
734
735 if (redis_fstat(server.appendfd,&sb) == -1) {
736 redisLog(REDIS_WARNING,"Unable to check the AOF length: %s",
737 strerror(errno));
738 } else {
739 server.appendonly_current_size = sb.st_size;
740 }
741 }
742
743 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
744 * Handle this. */
745 void backgroundRewriteDoneHandler(int statloc) {
746 int exitcode = WEXITSTATUS(statloc);
747 int bysignal = WIFSIGNALED(statloc);
748
749 if (!bysignal && exitcode == 0) {
750 int newfd, oldfd;
751 int nwritten;
752 char tmpfile[256];
753 long long now = ustime();
754
755 redisLog(REDIS_NOTICE,
756 "Background AOF rewrite terminated with success");
757
758 /* Flush the differences accumulated by the parent to the
759 * rewritten AOF. */
760 snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
761 (int)server.bgrewritechildpid);
762 newfd = open(tmpfile,O_WRONLY|O_APPEND);
763 if (newfd == -1) {
764 redisLog(REDIS_WARNING,
765 "Unable to open the temporary AOF produced by the child: %s", strerror(errno));
766 goto cleanup;
767 }
768
769 nwritten = write(newfd,server.bgrewritebuf,sdslen(server.bgrewritebuf));
770 if (nwritten != (signed)sdslen(server.bgrewritebuf)) {
771 if (nwritten == -1) {
772 redisLog(REDIS_WARNING,
773 "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
774 } else {
775 redisLog(REDIS_WARNING,
776 "Short write trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
777 }
778 close(newfd);
779 goto cleanup;
780 }
781
782 redisLog(REDIS_NOTICE,
783 "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", nwritten);
784
785 /* The only remaining thing to do is to rename the temporary file to
786 * the configured file and switch the file descriptor used to do AOF
787 * writes. We don't want close(2) or rename(2) calls to block the
788 * server on old file deletion.
789 *
790 * There are two possible scenarios:
791 *
792 * 1) AOF is DISABLED and this was a one time rewrite. The temporary
793 * file will be renamed to the configured file. When this file already
794 * exists, it will be unlinked, which may block the server.
795 *
796 * 2) AOF is ENABLED and the rewritten AOF will immediately start
797 * receiving writes. After the temporary file is renamed to the
798 * configured file, the original AOF file descriptor will be closed.
799 * Since this will be the last reference to that file, closing it
800 * causes the underlying file to be unlinked, which may block the
801 * server.
802 *
803 * To mitigate the blocking effect of the unlink operation (either
804 * caused by rename(2) in scenario 1, or by close(2) in scenario 2), we
805 * use a background thread to take care of this. First, we
806 * make scenario 1 identical to scenario 2 by opening the target file
807 * when it exists. The unlink operation after the rename(2) will then
808 * be executed upon calling close(2) for its descriptor. Everything to
809 * guarantee atomicity for this switch has already happened by then, so
810 * we don't care what the outcome or duration of that close operation
811 * is, as long as the file descriptor is released again. */
812 if (server.appendfd == -1) {
813 /* AOF disabled */
814
815 /* Don't care if this fails: oldfd will be -1 and we handle that.
816 * One notable case of -1 return is if the old file does
817 * not exist. */
818 oldfd = open(server.appendfilename,O_RDONLY|O_NONBLOCK);
819 } else {
820 /* AOF enabled */
821 oldfd = -1; /* We'll set this to the current AOF filedes later. */
822 }
823
824 /* Rename the temporary file. This will not unlink the target file if
825 * it exists, because we reference it with "oldfd". */
826 if (rename(tmpfile,server.appendfilename) == -1) {
827 redisLog(REDIS_WARNING,
828 "Error trying to rename the temporary AOF: %s", strerror(errno));
829 close(newfd);
830 if (oldfd != -1) close(oldfd);
831 goto cleanup;
832 }
833
834 if (server.appendfd == -1) {
835 /* AOF disabled, we don't need to set the AOF file descriptor
836 * to this new file, so we can close it. */
837 close(newfd);
838 } else {
839 /* AOF enabled, replace the old fd with the new one. */
840 oldfd = server.appendfd;
841 server.appendfd = newfd;
842 if (server.appendfsync == APPENDFSYNC_ALWAYS)
843 aof_fsync(newfd);
844 else if (server.appendfsync == APPENDFSYNC_EVERYSEC)
845 aof_background_fsync(newfd);
846 server.appendseldb = -1; /* Make sure SELECT is re-issued */
847 aofUpdateCurrentSize();
848 server.auto_aofrewrite_base_size = server.appendonly_current_size;
849
850 /* Clear regular AOF buffer since its contents was just written to
851 * the new AOF from the background rewrite buffer. */
852 sdsfree(server.aofbuf);
853 server.aofbuf = sdsempty();
854 }
855
856 redisLog(REDIS_NOTICE, "Background AOF rewrite successful");
857
858 /* Asynchronously close the overwritten AOF. */
859 if (oldfd != -1) bioCreateBackgroundJob(REDIS_BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL);
860
861 redisLog(REDIS_VERBOSE,
862 "Background AOF rewrite signal handler took %lldus", ustime()-now);
863 } else if (!bysignal && exitcode != 0) {
864 redisLog(REDIS_WARNING,
865 "Background AOF rewrite terminated with error");
866 } else {
867 redisLog(REDIS_WARNING,
868 "Background AOF rewrite terminated by signal %d",
869 WTERMSIG(statloc));
870 }
871
872 cleanup:
873 sdsfree(server.bgrewritebuf);
874 server.bgrewritebuf = sdsempty();
875 aofRemoveTempFile(server.bgrewritechildpid);
876 server.bgrewritechildpid = -1;
877 }