comparison src/readpst.c @ 202:2f38c4ce606f

remove readpstlog, switch to plain ascii debug log files
author Carl Byington <carl@five-ten-sg.com>
date Mon, 18 May 2009 15:55:05 -0700
parents 3850a3b11745
children 9fb600ef4e03
comparison
equal deleted inserted replaced
201:3850a3b11745 202:2f38c4ce606f
128 pid_t* child_processes; // setup by main(), and at the start of new child process 128 pid_t* child_processes; // setup by main(), and at the start of new child process
129 129
130 #ifdef HAVE_SEMAPHORE_H 130 #ifdef HAVE_SEMAPHORE_H
131 int shared_memory_id; 131 int shared_memory_id;
132 sem_t* global_children = NULL; 132 sem_t* global_children = NULL;
133 sem_t* output_mutex = NULL;
133 #endif 134 #endif
134 135
135 136
136 int grim_reaper(int waitall) 137 int grim_reaper(int waitall)
137 { 138 {
139 #ifdef HAVE_FORK 140 #ifdef HAVE_FORK
140 #ifdef HAVE_SEMAPHORE_H 141 #ifdef HAVE_SEMAPHORE_H
141 if (global_children) { 142 if (global_children) {
142 sem_getvalue(global_children, &available); 143 sem_getvalue(global_children, &available);
143 //printf("grim reaper %s for pid %d (parent %d) with %d children, %d available\n", (waitall) ? "all" : "", getpid(), getppid(), active_children, available); 144 //printf("grim reaper %s for pid %d (parent %d) with %d children, %d available\n", (waitall) ? "all" : "", getpid(), getppid(), active_children, available);
144 fflush(stdout); 145 //fflush(stdout);
145 int i,j; 146 int i,j;
146 for (i=0; i<active_children; i++) { 147 for (i=0; i<active_children; i++) {
147 pid_t child = child_processes[i]; 148 pid_t child = child_processes[i];
148 pid_t ch = waitpid(child, NULL, ((waitall) ? 0 : WNOHANG)); 149 pid_t ch = waitpid(child, NULL, ((waitall) ? 0 : WNOHANG));
149 if (ch == child) { 150 if (ch == child) {
155 i--; 156 i--;
156 } 157 }
157 } 158 }
158 sem_getvalue(global_children, &available); 159 sem_getvalue(global_children, &available);
159 //printf("grim reaper %s for pid %d with %d children, %d available\n", (waitall) ? "all" : "", getpid(), active_children, available); 160 //printf("grim reaper %s for pid %d with %d children, %d available\n", (waitall) ? "all" : "", getpid(), active_children, available);
160 fflush(stdout); 161 //fflush(stdout);
161 } 162 }
162 #endif 163 #endif
163 #endif 164 #endif
164 return available; 165 return available;
165 } 166 }
185 } 186 }
186 else { 187 else {
187 // fork worked, and we are the parent, record this child that we need to wait for 188 // fork worked, and we are the parent, record this child that we need to wait for
188 pid_t me = getpid(); 189 pid_t me = getpid();
189 //printf("parent %d forked child pid %d to process folder %s\n", me, child, folder); 190 //printf("parent %d forked child pid %d to process folder %s\n", me, child, folder);
190 fflush(stdout); 191 //fflush(stdout);
191 child_processes[active_children++] = child; 192 child_processes[active_children++] = child;
192 } 193 }
193 return child; 194 return child;
194 } 195 }
195 else { 196 else {
209 DEBUG_ENT("process"); 210 DEBUG_ENT("process");
210 memset(&ff, 0, sizeof(ff)); 211 memset(&ff, 0, sizeof(ff));
211 create_enter_dir(&ff, outeritem); 212 create_enter_dir(&ff, outeritem);
212 213
213 for (; d_ptr; d_ptr = d_ptr->next) { 214 for (; d_ptr; d_ptr = d_ptr->next) {
214 DEBUG_MAIN(("main: New item record\n")); 215 DEBUG_INFO(("New item record\n"));
215 if (!d_ptr->desc) { 216 if (!d_ptr->desc) {
216 ff.skip_count++; 217 ff.skip_count++;
217 DEBUG_WARN(("main: ERROR item's desc record is NULL\n")); 218 DEBUG_WARN(("ERROR item's desc record is NULL\n"));
218 continue; 219 continue;
219 } 220 }
220 DEBUG_MAIN(("main: Desc Email ID %#"PRIx64" [d_ptr->d_id = %#"PRIx64"]\n", d_ptr->desc->i_id, d_ptr->d_id)); 221 DEBUG_INFO(("Desc Email ID %#"PRIx64" [d_ptr->d_id = %#"PRIx64"]\n", d_ptr->desc->i_id, d_ptr->d_id));
221 222
222 item = pst_parse_item(&pstfile, d_ptr, NULL); 223 item = pst_parse_item(&pstfile, d_ptr, NULL);
223 DEBUG_MAIN(("main: About to process item\n")); 224 DEBUG_INFO(("About to process item\n"));
224 225
225 if (!item) { 226 if (!item) {
226 ff.skip_count++; 227 ff.skip_count++;
227 DEBUG_MAIN(("main: A NULL item was seen\n")); 228 DEBUG_INFO(("A NULL item was seen\n"));
228 continue; 229 continue;
229 } 230 }
230 231
231 if (item->subject.str) { 232 if (item->subject.str) {
232 DEBUG_EMAIL(("item->subject = %s\n", item->subject.str)); 233 DEBUG_INFO(("item->subject = %s\n", item->subject.str));
233 } 234 }
234 235
235 if (item->folder && item->file_as.str) { 236 if (item->folder && item->file_as.str) {
236 DEBUG_MAIN(("Processing Folder \"%s\"\n", item->file_as.str)); 237 DEBUG_INFO(("Processing Folder \"%s\"\n", item->file_as.str));
237 if (output_mode != OUTPUT_QUIET) printf("Processing Folder \"%s\"\n", item->file_as.str); 238 if (output_mode != OUTPUT_QUIET) {
238 fflush(stdout); 239 pst_debug_lock();
240 printf("Processing Folder \"%s\"\n", item->file_as.str);
241 fflush(stdout);
242 pst_debug_unlock();
243 }
239 ff.item_count++; 244 ff.item_count++;
240 if (d_ptr->child && (deleted_mode == DMODE_INCLUDE || strcasecmp(item->file_as.str, "Deleted Items"))) { 245 if (d_ptr->child && (deleted_mode == DMODE_INCLUDE || strcasecmp(item->file_as.str, "Deleted Items"))) {
241 //if this is a non-empty folder other than deleted items, we want to recurse into it 246 //if this is a non-empty folder other than deleted items, we want to recurse into it
242 pid_t parent = getpid(); 247 pid_t parent = getpid();
243 pid_t child = try_fork(item->file_as.str); 248 pid_t child = try_fork(item->file_as.str);
260 } 265 }
261 } 266 }
262 267
263 } else if (item->contact && (item->type == PST_TYPE_CONTACT)) { 268 } else if (item->contact && (item->type == PST_TYPE_CONTACT)) {
264 if (!ff.type) ff.type = item->type; 269 if (!ff.type) ff.type = item->type;
265 DEBUG_MAIN(("main: Processing Contact\n")); 270 DEBUG_INFO(("Processing Contact\n"));
266 if (ff.type != PST_TYPE_CONTACT) { 271 if (ff.type != PST_TYPE_CONTACT) {
267 ff.skip_count++; 272 ff.skip_count++;
268 DEBUG_MAIN(("main: I have a contact, but the folder type %"PRIi32" isn't a contacts folder. Skipping it\n", ff.type)); 273 DEBUG_INFO(("I have a contact, but the folder type %"PRIi32" isn't a contacts folder. Skipping it\n", ff.type));
269 } 274 }
270 else { 275 else {
271 ff.item_count++; 276 ff.item_count++;
272 if (mode == MODE_SEPARATE) mk_separate_file(&ff); 277 if (mode == MODE_SEPARATE) mk_separate_file(&ff);
273 if (contact_mode == CMODE_VCARD) { 278 if (contact_mode == CMODE_VCARD) {
281 } 286 }
282 } 287 }
283 288
284 } else if (item->email && ((item->type == PST_TYPE_NOTE) || (item->type == PST_TYPE_SCHEDULE) || (item->type == PST_TYPE_REPORT))) { 289 } else if (item->email && ((item->type == PST_TYPE_NOTE) || (item->type == PST_TYPE_SCHEDULE) || (item->type == PST_TYPE_REPORT))) {
285 if (!ff.type) ff.type = item->type; 290 if (!ff.type) ff.type = item->type;
286 DEBUG_MAIN(("main: Processing Email\n")); 291 DEBUG_INFO(("Processing Email\n"));
287 if ((ff.type != PST_TYPE_NOTE) && (ff.type != PST_TYPE_SCHEDULE) && (ff.type != PST_TYPE_REPORT)) { 292 if ((ff.type != PST_TYPE_NOTE) && (ff.type != PST_TYPE_SCHEDULE) && (ff.type != PST_TYPE_REPORT)) {
288 ff.skip_count++; 293 ff.skip_count++;
289 DEBUG_MAIN(("main: I have an email type %"PRIi32", but the folder type %"PRIi32" isn't an email folder. Skipping it\n", item->type, ff.type)); 294 DEBUG_INFO(("I have an email type %"PRIi32", but the folder type %"PRIi32" isn't an email folder. Skipping it\n", item->type, ff.type));
290 } 295 }
291 else { 296 else {
292 char *extra_mime_headers = NULL; 297 char *extra_mime_headers = NULL;
293 ff.item_count++; 298 ff.item_count++;
294 if (mode == MODE_SEPARATE) mk_separate_file(&ff); 299 if (mode == MODE_SEPARATE) mk_separate_file(&ff);
295 write_normal_email(ff.output, ff.name, item, mode, mode_MH, &pstfile, save_rtf_body, &extra_mime_headers); 300 write_normal_email(ff.output, ff.name, item, mode, mode_MH, &pstfile, save_rtf_body, &extra_mime_headers);
296 } 301 }
297 302
298 } else if (item->journal && (item->type == PST_TYPE_JOURNAL)) { 303 } else if (item->journal && (item->type == PST_TYPE_JOURNAL)) {
299 if (!ff.type) ff.type = item->type; 304 if (!ff.type) ff.type = item->type;
300 DEBUG_MAIN(("main: Processing Journal Entry\n")); 305 DEBUG_INFO(("Processing Journal Entry\n"));
301 if (ff.type != PST_TYPE_JOURNAL) { 306 if (ff.type != PST_TYPE_JOURNAL) {
302 ff.skip_count++; 307 ff.skip_count++;
303 DEBUG_MAIN(("main: I have a journal entry, but the folder type %"PRIi32" isn't a journal folder. Skipping it\n", ff.type)); 308 DEBUG_INFO(("I have a journal entry, but the folder type %"PRIi32" isn't a journal folder. Skipping it\n", ff.type));
304 } 309 }
305 else { 310 else {
306 ff.item_count++; 311 ff.item_count++;
307 if (mode == MODE_SEPARATE) mk_separate_file(&ff); 312 if (mode == MODE_SEPARATE) mk_separate_file(&ff);
308 write_journal(ff.output, item); 313 write_journal(ff.output, item);
309 fprintf(ff.output, "\n"); 314 fprintf(ff.output, "\n");
310 } 315 }
311 316
312 } else if (item->appointment && (item->type == PST_TYPE_APPOINTMENT)) { 317 } else if (item->appointment && (item->type == PST_TYPE_APPOINTMENT)) {
313 if (!ff.type) ff.type = item->type; 318 if (!ff.type) ff.type = item->type;
314 DEBUG_MAIN(("main: Processing Appointment Entry\n")); 319 DEBUG_INFO(("Processing Appointment Entry\n"));
315 if (ff.type != PST_TYPE_APPOINTMENT) { 320 if (ff.type != PST_TYPE_APPOINTMENT) {
316 ff.skip_count++; 321 ff.skip_count++;
317 DEBUG_MAIN(("main: I have an appointment, but the folder type %"PRIi32" isn't an appointment folder. Skipping it\n", ff.type)); 322 DEBUG_INFO(("I have an appointment, but the folder type %"PRIi32" isn't an appointment folder. Skipping it\n", ff.type));
318 } 323 }
319 else { 324 else {
320 ff.item_count++; 325 ff.item_count++;
321 if (mode == MODE_SEPARATE) mk_separate_file(&ff); 326 if (mode == MODE_SEPARATE) mk_separate_file(&ff);
322 write_appointment(ff.output, item, 0); 327 write_appointment(ff.output, item, 0);
324 } 329 }
325 330
326 } else if (item->message_store) { 331 } else if (item->message_store) {
327 // there should only be one message_store, and we have already done it 332 // there should only be one message_store, and we have already done it
328 ff.skip_count++; 333 ff.skip_count++;
329 DEBUG_MAIN(("item with message store content, type %i %s folder type %i, skipping it\n", item->type, item->ascii_type, ff.type)); 334 DEBUG_INFO(("item with message store content, type %i %s folder type %i, skipping it\n", item->type, item->ascii_type, ff.type));
330 335
331 } else { 336 } else {
332 ff.skip_count++; 337 ff.skip_count++;
333 DEBUG_MAIN(("main: Unknown item type %i (%s) name (%s)\n", 338 DEBUG_INFO(("Unknown item type %i (%s) name (%s)\n",
334 item->type, item->ascii_type, item->file_as.str)); 339 item->type, item->ascii_type, item->file_as.str));
335 } 340 }
336 pst_freeItem(item); 341 pst_freeItem(item);
337 } 342 }
338 close_enter_dir(&ff); 343 close_enter_dir(&ff);
431 } else { 436 } else {
432 usage(); 437 usage();
433 exit(2); 438 exit(2);
434 } 439 }
435 440
441 #ifdef _SC_NPROCESSORS_ONLN
442 number_processors = sysconf(_SC_NPROCESSORS_ONLN);
443 #endif
444 max_children = (max_child_specified) ? max_children : number_processors * 4;
445 active_children = 0;
446 child_processes = (pid_t *)pst_malloc(sizeof(pid_t) * max_children);
447 memset(child_processes, 0, sizeof(pid_t) * max_children);
448
449 #ifdef HAVE_SEMAPHORE_H
450 if (max_children) {
451 shared_memory_id = shmget(IPC_PRIVATE, sizeof(sem_t)*2, 0777);
452 if (shared_memory_id >= 0) {
453 global_children = (sem_t *)shmat(shared_memory_id, NULL, 0);
454 if (global_children == (sem_t *)-1) global_children = NULL;
455 if (global_children) {
456 output_mutex = &(global_children[1]);
457 sem_init(global_children, 1, max_children);
458 sem_init(output_mutex, 1, 1);
459 }
460 shmctl(shared_memory_id, IPC_RMID, NULL);
461 }
462 }
463 #endif
464
436 #ifdef DEBUG_ALL 465 #ifdef DEBUG_ALL
437 // force a log file 466 // force a log file
438 if (!d_log) d_log = "readpst.log"; 467 if (!d_log) d_log = "readpst.log";
439 #endif // defined DEBUG_ALL 468 #endif // defined DEBUG_ALL
440 DEBUG_INIT(d_log); 469 DEBUG_INIT(d_log, output_mutex);
441 DEBUG_REGISTER_CLOSE();
442 DEBUG_ENT("main"); 470 DEBUG_ENT("main");
443 471
444 if (output_mode != OUTPUT_QUIET) printf("Opening PST file and indexes...\n"); 472 if (output_mode != OUTPUT_QUIET) printf("Opening PST file and indexes...\n");
445 473
446 RET_DERROR(pst_open(&pstfile, fname), 1, ("Error opening File\n")); 474 RET_DERROR(pst_open(&pstfile, fname), 1, ("Error opening File\n"));
450 478
451 if (chdir(output_dir)) { 479 if (chdir(output_dir)) {
452 x = errno; 480 x = errno;
453 pst_close(&pstfile); 481 pst_close(&pstfile);
454 DEBUG_RET(); 482 DEBUG_RET();
455 DIE(("main: Cannot change to output dir %s: %s\n", output_dir, strerror(x))); 483 DIE(("Cannot change to output dir %s: %s\n", output_dir, strerror(x)));
456 } 484 }
457 485
458 d_ptr = pstfile.d_head; // first record is main record 486 d_ptr = pstfile.d_head; // first record is main record
459 item = pst_parse_item(&pstfile, d_ptr, NULL); 487 item = pst_parse_item(&pstfile, d_ptr, NULL);
460 if (!item || !item->message_store) { 488 if (!item || !item->message_store) {
461 DEBUG_RET(); 489 DEBUG_RET();
462 DIE(("main: Could not get root record\n")); 490 DIE(("Could not get root record\n"));
463 } 491 }
464 492
465 // default the file_as to the same as the main filename if it doesn't exist 493 // default the file_as to the same as the main filename if it doesn't exist
466 if (!item->file_as.str) { 494 if (!item->file_as.str) {
467 if (!(temp = strrchr(fname, '/'))) 495 if (!(temp = strrchr(fname, '/')))
472 else 500 else
473 temp++; // get past the "/" 501 temp++; // get past the "/"
474 item->file_as.str = (char*)pst_malloc(strlen(temp)+1); 502 item->file_as.str = (char*)pst_malloc(strlen(temp)+1);
475 strcpy(item->file_as.str, temp); 503 strcpy(item->file_as.str, temp);
476 item->file_as.is_utf8 = 1; 504 item->file_as.is_utf8 = 1;
477 DEBUG_MAIN(("file_as was blank, so am using %s\n", item->file_as.str)); 505 DEBUG_INFO(("file_as was blank, so am using %s\n", item->file_as.str));
478 } 506 }
479 DEBUG_MAIN(("main: Root Folder Name: %s\n", item->file_as.str)); 507 DEBUG_INFO(("Root Folder Name: %s\n", item->file_as.str));
480 508
481 d_ptr = pst_getTopOfFolders(&pstfile, item); 509 d_ptr = pst_getTopOfFolders(&pstfile, item);
482 if (!d_ptr) { 510 if (!d_ptr) {
483 DEBUG_RET(); 511 DEBUG_RET();
484 DIE(("Top of folders record not found. Cannot continue\n")); 512 DIE(("Top of folders record not found. Cannot continue\n"));
485 } 513 }
486 514
487 #ifdef _SC_NPROCESSORS_ONLN
488 number_processors = sysconf(_SC_NPROCESSORS_ONLN);
489 #endif
490 max_children = (d_log) ? 0 : (!max_child_specified) ? number_processors * 4 : max_children;
491 active_children = 0;
492 child_processes = (pid_t *)pst_malloc(sizeof(pid_t) * max_children);
493 memset(child_processes, 0, sizeof(pid_t) * max_children);
494
495 #ifdef HAVE_SEMAPHORE_H
496 if (max_children) {
497 shared_memory_id = shmget(IPC_PRIVATE, sizeof(sem_t), 0777);
498 //printf("shared memory id %d\n", shared_memory_id);
499 if (shared_memory_id >= 0) {
500 global_children = (sem_t *)shmat(shared_memory_id, NULL, 0);
501 //printf("shared memory pointer %p\n", (void*)global_children);
502 if (global_children == (sem_t *)-1) global_children = NULL;
503 if (global_children) sem_init(global_children, 1, max_children);
504 shmctl(shared_memory_id, IPC_RMID, NULL);
505 }
506 }
507 #endif
508
509 process(item, d_ptr->child); // do the children of TOPF 515 process(item, d_ptr->child); // do the children of TOPF
510 grim_reaper(1); // wait for all child processes 516 grim_reaper(1); // wait for all child processes
517
518 pst_freeItem(item);
519 pst_close(&pstfile);
520 DEBUG_RET();
511 521
512 #ifdef HAVE_SEMAPHORE_H 522 #ifdef HAVE_SEMAPHORE_H
513 if (global_children) { 523 if (global_children) {
514 sem_destroy(global_children); 524 sem_destroy(global_children);
525 sem_destroy(output_mutex);
515 shmdt(global_children); 526 shmdt(global_children);
516 } 527 }
517 #endif 528 #endif
518 529
519 pst_freeItem(item);
520 pst_close(&pstfile);
521 DEBUG_RET();
522 regfree(&meta_charset_pattern); 530 regfree(&meta_charset_pattern);
523 return 0; 531 return 0;
524 } 532 }
525 533
526 534
527 void write_email_body(FILE *f, char *body) { 535 void write_email_body(FILE *f, char *body) {
528 char *n = body; 536 char *n = body;
529 DEBUG_ENT("write_email_body"); 537 DEBUG_ENT("write_email_body");
530 DEBUG_INFO(("buffer pointer %p\n", body));
531 while (n) { 538 while (n) {
532 if (strncmp(body, "From ", 5) == 0) 539 if (strncmp(body, "From ", 5) == 0)
533 fprintf(f, ">"); 540 fprintf(f, ">");
534 if ((n = strchr(body, '\n'))) { 541 if ((n = strchr(body, '\n'))) {
535 n++; 542 n++;
613 } 620 }
614 dir = malloc(strlen(fname)+strlen(OUTPUT_KMAIL_DIR_TEMPLATE)+1); 621 dir = malloc(strlen(fname)+strlen(OUTPUT_KMAIL_DIR_TEMPLATE)+1);
615 sprintf(dir, OUTPUT_KMAIL_DIR_TEMPLATE, fname); 622 sprintf(dir, OUTPUT_KMAIL_DIR_TEMPLATE, fname);
616 check_filename(dir); 623 check_filename(dir);
617 if (D_MKDIR(dir)) { 624 if (D_MKDIR(dir)) {
618 //error occured 625 if (errno != EEXIST) { // not an error because it exists
619 if (errno != EEXIST) {
620 x = errno; 626 x = errno;
621 DIE(("mk_kmail_dir: Cannot create directory %s: %s\n", dir, strerror(x))); 627 DIE(("mk_kmail_dir: Cannot create directory %s: %s\n", dir, strerror(x)));
622 } 628 }
623 } 629 }
624 kmail_chdir = realloc(kmail_chdir, strlen(dir)+1); 630 kmail_chdir = realloc(kmail_chdir, strlen(dir)+1);
654 DEBUG_RET(); 660 DEBUG_RET();
655 return 0; 661 return 0;
656 } 662 }
657 663
658 664
659 // this will create a directory by that name, then make an mbox file inside 665 // this will create a directory by that name,
660 // that dir. any subsequent dirs will be created by name, and they will 666 // then make an mbox file inside that directory.
661 // contain mbox files
662 char *mk_recurse_dir(char *dir, int32_t folder_type) { 667 char *mk_recurse_dir(char *dir, int32_t folder_type) {
663 int x; 668 int x;
664 char *out_name; 669 char *out_name;
665 DEBUG_ENT("mk_recurse_dir"); 670 DEBUG_ENT("mk_recurse_dir");
666 check_filename(dir); 671 check_filename(dir);
667 if (D_MKDIR (dir)) { 672 if (D_MKDIR (dir)) {
668 if (errno != EEXIST) { // not an error because it exists 673 if (errno != EEXIST) { // not an error because it exists
669 x = errno; 674 x = errno;
670 DIE(("mk_recurse_dir: Cannot create directory %s: %s\n", dir, strerror(x))); 675 DIE(("mk_recurse_dir: Cannot create directory %s: %s\n", dir, strerror(x)));
671 } 676 }
672 } 677 }
673 if (chdir (dir)) { 678 if (chdir (dir)) {
721 snprintf(dir_name, dirsize, "%s", dir); 726 snprintf(dir_name, dirsize, "%s", dir);
722 else 727 else
723 snprintf(dir_name, dirsize, "%s" SEP_MAIL_FILE_TEMPLATE, dir, y); // enough for 9 digits allocated above 728 snprintf(dir_name, dirsize, "%s" SEP_MAIL_FILE_TEMPLATE, dir, y); // enough for 9 digits allocated above
724 729
725 check_filename(dir_name); 730 check_filename(dir_name);
726 DEBUG_MAIN(("about to try creating %s\n", dir_name)); 731 DEBUG_INFO(("about to try creating %s\n", dir_name));
727 if (D_MKDIR(dir_name)) { 732 if (D_MKDIR(dir_name)) {
728 if (errno != EEXIST) { // if there is an error, and it doesn't already exist 733 if (errno != EEXIST) { // if there is an error, and it doesn't already exist
729 x = errno; 734 x = errno;
730 DIE(("mk_separate_dir: Cannot create directory %s: %s\n", dir, strerror(x))); 735 DIE(("mk_separate_dir: Cannot create directory %s: %s\n", dir, strerror(x)));
731 } 736 }
781 786
782 787
783 int mk_separate_file(struct file_ll *f) { 788 int mk_separate_file(struct file_ll *f) {
784 const int name_offset = 1; 789 const int name_offset = 1;
785 DEBUG_ENT("mk_separate_file"); 790 DEBUG_ENT("mk_separate_file");
786 DEBUG_MAIN(("opening next file to save email\n")); 791 DEBUG_INFO(("opening next file to save email\n"));
787 if (f->item_count > 999999999) { // bigger than nine 9's 792 if (f->item_count > 999999999) { // bigger than nine 9's
788 DIE(("mk_separate_file: The number of emails in this folder has become too high to handle\n")); 793 DIE(("mk_separate_file: The number of emails in this folder has become too high to handle\n"));
789 } 794 }
790 sprintf(f->name, SEP_MAIL_FILE_TEMPLATE, f->item_count + name_offset); 795 sprintf(f->name, SEP_MAIL_FILE_TEMPLATE, f->item_count + name_offset);
791 if (f->output) fclose(f->output); 796 if (f->output) fclose(f->output);
878 } while ((fp = fopen(temp, "r")) && ++x < 99999999); 883 } while ((fp = fopen(temp, "r")) && ++x < 99999999);
879 if (x > 99999999) { 884 if (x > 99999999) {
880 DIE(("error finding attachment name. exhausted possibilities to %s\n", temp)); 885 DIE(("error finding attachment name. exhausted possibilities to %s\n", temp));
881 } 886 }
882 } 887 }
883 DEBUG_EMAIL(("Saving attachment to %s\n", temp)); 888 DEBUG_INFO(("Saving attachment to %s\n", temp));
884 if (!(fp = fopen(temp, "w"))) { 889 if (!(fp = fopen(temp, "w"))) {
885 DEBUG_WARN(("write_separate_attachment: Cannot open attachment save file \"%s\"\n", temp)); 890 DEBUG_WARN(("write_separate_attachment: Cannot open attachment save file \"%s\"\n", temp));
886 } else { 891 } else {
887 (void)pst_attach_to_file(pst, attach, fp); 892 (void)pst_attach_to_file(pst, attach, fp);
888 fclose(fp); 893 fclose(fp);
922 927
923 void write_inline_attachment(FILE* f_output, pst_item_attach* attach, char *boundary, pst_file* pst) 928 void write_inline_attachment(FILE* f_output, pst_item_attach* attach, char *boundary, pst_file* pst)
924 { 929 {
925 char *attach_filename; 930 char *attach_filename;
926 DEBUG_ENT("write_inline_attachment"); 931 DEBUG_ENT("write_inline_attachment");
927 DEBUG_EMAIL(("Attachment Size is %"PRIu64", id %#"PRIx64"\n", (uint64_t)attach->data.size, attach->i_id)); 932 DEBUG_INFO(("Attachment Size is %"PRIu64", id %#"PRIx64"\n", (uint64_t)attach->data.size, attach->i_id));
928 933
929 if (!attach->data.data) { 934 if (!attach->data.data) {
930 // make sure we can fetch data from the id 935 // make sure we can fetch data from the id
931 pst_index_ll *ptr = pst_getID(pst, attach->i_id); 936 pst_index_ll *ptr = pst_getID(pst, attach->i_id);
932 if (!ptr) { 937 if (!ptr) {
961 966
962 void header_has_field(char *header, char *field, int *flag) 967 void header_has_field(char *header, char *field, int *flag)
963 { 968 {
964 DEBUG_ENT("header_has_field"); 969 DEBUG_ENT("header_has_field");
965 if (my_stristr(header, field) || (strncasecmp(header, field+1, strlen(field)-1) == 0)) { 970 if (my_stristr(header, field) || (strncasecmp(header, field+1, strlen(field)-1) == 0)) {
966 DEBUG_EMAIL(("header block has %s header\n", field+1)); 971 DEBUG_INFO(("header block has %s header\n", field+1));
967 *flag = 1; 972 *flag = 1;
968 } 973 }
969 DEBUG_RET(); 974 DEBUG_RET();
970 } 975 }
971 976
994 if (!e || (e > n)) e = n; // use the trailing lf as terminator if nothing better 999 if (!e || (e > n)) e = n; // use the trailing lf as terminator if nothing better
995 save = *e; 1000 save = *e;
996 *e = '\0'; 1001 *e = '\0';
997 snprintf(body_subfield, size_subfield, "%s", s); // copy the subfield to our buffer 1002 snprintf(body_subfield, size_subfield, "%s", s); // copy the subfield to our buffer
998 *e = save; 1003 *e = save;
999 DEBUG_EMAIL(("body %s %s from headers\n", subfield, body_subfield)); 1004 DEBUG_INFO(("body %s %s from headers\n", subfield, body_subfield));
1000 } 1005 }
1001 DEBUG_RET(); 1006 DEBUG_RET();
1002 } 1007 }
1003 1008
1004 char* header_get_field(char *header, char *field) 1009 char* header_get_field(char *header, char *field)
1048 int b64 = 0; 1053 int b64 = 0;
1049 uint8_t *b = (uint8_t *)body; 1054 uint8_t *b = (uint8_t *)body;
1050 DEBUG_ENT("test_base64"); 1055 DEBUG_ENT("test_base64");
1051 while (*b != 0) { 1056 while (*b != 0) {
1052 if ((*b < 32) && (*b != 9) && (*b != 10)) { 1057 if ((*b < 32) && (*b != 9) && (*b != 10)) {
1053 DEBUG_EMAIL(("found base64 byte %d\n", (int)*b)); 1058 DEBUG_INFO(("found base64 byte %d\n", (int)*b));
1054 DEBUG_HEXDUMPC(body, strlen(body), 0x10); 1059 DEBUG_HEXDUMPC(body, strlen(body), 0x10);
1055 b64 = 1; 1060 b64 = 1;
1056 break; 1061 break;
1057 } 1062 }
1058 b++; 1063 b++;
1075 if (s != -1) { 1080 if (s != -1) {
1076 char save = html[e]; 1081 char save = html[e];
1077 html[e] = '\0'; 1082 html[e] = '\0';
1078 snprintf(charset, charsetlen, "%s", html+s); // copy the html charset 1083 snprintf(charset, charsetlen, "%s", html+s); // copy the html charset
1079 html[e] = save; 1084 html[e] = save;
1080 DEBUG_EMAIL(("charset %s from html text\n", charset)); 1085 DEBUG_INFO(("charset %s from html text\n", charset));
1081 } 1086 }
1082 else { 1087 else {
1083 DEBUG_EMAIL(("matching %d %d %d %d", match[0].rm_so, match[0].rm_eo, match[1].rm_so, match[1].rm_eo)); 1088 DEBUG_INFO(("matching %d %d %d %d", match[0].rm_so, match[0].rm_eo, match[1].rm_so, match[1].rm_eo));
1084 DEBUG_HEXDUMPC(html, strlen(html), 0x10); 1089 DEBUG_HEXDUMPC(html, strlen(html), 0x10);
1085 } 1090 }
1086 } 1091 }
1087 else { 1092 else {
1088 DEBUG_EMAIL(("regexec returns %d\n", rc)); 1093 DEBUG_INFO(("regexec returns %d\n", rc));
1089 } 1094 }
1090 DEBUG_RET(); 1095 DEBUG_RET();
1091 } 1096 }
1092 1097
1093 1098
1100 while ((temp = strstr(headers, "\n\n"))) { 1105 while ((temp = strstr(headers, "\n\n"))) {
1101 temp[1] = '\0'; 1106 temp[1] = '\0';
1102 t = header_get_field(headers, "\nContent-Type: "); 1107 t = header_get_field(headers, "\nContent-Type: ");
1103 if (t) { 1108 if (t) {
1104 t++; 1109 t++;
1105 DEBUG_EMAIL(("found content type header\n")); 1110 DEBUG_INFO(("found content type header\n"));
1106 char *n = strchr(t, '\n'); 1111 char *n = strchr(t, '\n');
1107 char *s = strstr(t, ": "); 1112 char *s = strstr(t, ": ");
1108 char *e = strchr(t, ';'); 1113 char *e = strchr(t, ';');
1109 if (!e || (e > n)) e = n; 1114 if (!e || (e > n)) e = n;
1110 if (s && (s < e)) { 1115 if (s && (s < e)) {
1111 s += 2; 1116 s += 2;
1112 if (!strncasecmp(s, RFC822, e-s)) { 1117 if (!strncasecmp(s, RFC822, e-s)) {
1113 headers = temp+2; // found rfc822 header 1118 headers = temp+2; // found rfc822 header
1114 DEBUG_EMAIL(("found 822 headers\n%s\n", headers)); 1119 DEBUG_INFO(("found 822 headers\n%s\n", headers));
1115 break; 1120 break;
1116 } 1121 }
1117 } 1122 }
1118 } 1123 }
1119 //DEBUG_EMAIL(("skipping to next block after\n%s\n", headers)); 1124 //DEBUG_INFO(("skipping to next block after\n%s\n", headers));
1120 headers = temp+2; // skip to next chunk of headers 1125 headers = temp+2; // skip to next chunk of headers
1121 } 1126 }
1122 *extra_mime_headers = headers; 1127 *extra_mime_headers = headers;
1123 } 1128 }
1124 DEBUG_RET(); 1129 DEBUG_RET();
1131 if (body->is_utf8 && (strcasecmp("utf-8", charset))) { 1136 if (body->is_utf8 && (strcasecmp("utf-8", charset))) {
1132 // try to convert to the specified charset since the target 1137 // try to convert to the specified charset since the target
1133 // is not utf-8, and the data came from a unicode (utf16) field 1138 // is not utf-8, and the data came from a unicode (utf16) field
1134 // and is now in utf-8. 1139 // and is now in utf-8.
1135 size_t rc; 1140 size_t rc;
1136 DEBUG_EMAIL(("Convert %s utf-8 to %s\n", mime, charset)); 1141 DEBUG_INFO(("Convert %s utf-8 to %s\n", mime, charset));
1137 pst_vbuf *newer = pst_vballoc(2); 1142 pst_vbuf *newer = pst_vballoc(2);
1138 rc = pst_vb_utf8to8bit(newer, body->str, strlen(body->str), charset); 1143 rc = pst_vb_utf8to8bit(newer, body->str, strlen(body->str), charset);
1139 if (rc == (size_t)-1) { 1144 if (rc == (size_t)-1) {
1140 // unable to convert, change the charset to utf8 1145 // unable to convert, change the charset to utf8
1141 free(newer->b); 1146 free(newer->b);
1142 DEBUG_EMAIL(("Failed to convert %s utf-8 to %s\n", mime, charset)); 1147 DEBUG_INFO(("Failed to convert %s utf-8 to %s\n", mime, charset));
1143 charset = "utf-8"; 1148 charset = "utf-8";
1144 } 1149 }
1145 else { 1150 else {
1146 // null terminate the output string 1151 // null terminate the output string
1147 pst_vbgrow(newer, 1); 1152 pst_vbgrow(newer, 1);
1267 // cut off our real rfc822 headers here 1272 // cut off our real rfc822 headers here
1268 temp[1] = '\0'; 1273 temp[1] = '\0';
1269 // pointer to all the embedded MIME headers. 1274 // pointer to all the embedded MIME headers.
1270 // we use these to find the actual rfc822 headers for embedded message/rfc822 mime parts 1275 // we use these to find the actual rfc822 headers for embedded message/rfc822 mime parts
1271 *extra_mime_headers = temp+2; 1276 *extra_mime_headers = temp+2;
1272 DEBUG_EMAIL(("Found extra mime headers\n%s\n", temp+2)); 1277 DEBUG_INFO(("Found extra mime headers\n%s\n", temp+2));
1273 } 1278 }
1274 1279
1275 // Check if the headers have all the necessary fields 1280 // Check if the headers have all the necessary fields
1276 header_has_field(headers, "\nFrom: ", &has_from); 1281 header_has_field(headers, "\nFrom: ", &has_from);
1277 header_has_field(headers, "\nTo: ", &has_to); 1282 header_has_field(headers, "\nTo: ", &has_to);
1312 header_strip_field(headers, "\nX-MimeOLE: "); 1317 header_strip_field(headers, "\nX-MimeOLE: ");
1313 header_strip_field(headers, "\nBcc:"); 1318 header_strip_field(headers, "\nBcc:");
1314 header_strip_field(headers, "\nX-From_: "); 1319 header_strip_field(headers, "\nX-From_: ");
1315 } 1320 }
1316 1321
1317 DEBUG_EMAIL(("About to print Header\n")); 1322 DEBUG_INFO(("About to print Header\n"));
1318 1323
1319 if (item && item->subject.str) { 1324 if (item && item->subject.str) {
1320 pst_convert_utf8(item, &item->subject); 1325 pst_convert_utf8(item, &item->subject);
1321 DEBUG_EMAIL(("item->subject = %s\n", item->subject.str)); 1326 DEBUG_INFO(("item->subject = %s\n", item->subject.str));
1322 } 1327 }
1323 1328
1324 if (mode != MODE_SEPARATE) { 1329 if (mode != MODE_SEPARATE) {
1325 // most modes need this separator line. 1330 // most modes need this separator line.
1326 // procmail produces this separator without the quotes around the 1331 // procmail produces this separator without the quotes around the
1421 write_body_part(f_output, &item->email->htmlbody, "text/html", body_charset, boundary, pst); 1426 write_body_part(f_output, &item->email->htmlbody, "text/html", body_charset, boundary, pst);
1422 } 1427 }
1423 1428
1424 if (item->email->rtf_compressed.data && save_rtf) { 1429 if (item->email->rtf_compressed.data && save_rtf) {
1425 pst_item_attach* attach = (pst_item_attach*)pst_malloc(sizeof(pst_item_attach)); 1430 pst_item_attach* attach = (pst_item_attach*)pst_malloc(sizeof(pst_item_attach));
1426 DEBUG_EMAIL(("Adding RTF body as attachment\n")); 1431 DEBUG_INFO(("Adding RTF body as attachment\n"));
1427 memset(attach, 0, sizeof(pst_item_attach)); 1432 memset(attach, 0, sizeof(pst_item_attach));
1428 attach->next = item->attach; 1433 attach->next = item->attach;
1429 item->attach = attach; 1434 item->attach = attach;
1430 attach->data.data = pst_lzfu_decompress(item->email->rtf_compressed.data, item->email->rtf_compressed.size, &attach->data.size); 1435 attach->data.data = pst_lzfu_decompress(item->email->rtf_compressed.data, item->email->rtf_compressed.size, &attach->data.size);
1431 attach->filename2.str = strdup(RTF_ATTACH_NAME); 1436 attach->filename2.str = strdup(RTF_ATTACH_NAME);
1436 1441
1437 if (item->email->encrypted_body.data || item->email->encrypted_htmlbody.data) { 1442 if (item->email->encrypted_body.data || item->email->encrypted_htmlbody.data) {
1438 // if either the body or htmlbody is encrypted, add them as attachments 1443 // if either the body or htmlbody is encrypted, add them as attachments
1439 if (item->email->encrypted_body.data) { 1444 if (item->email->encrypted_body.data) {
1440 pst_item_attach* attach = (pst_item_attach*)pst_malloc(sizeof(pst_item_attach)); 1445 pst_item_attach* attach = (pst_item_attach*)pst_malloc(sizeof(pst_item_attach));
1441 DEBUG_EMAIL(("Adding Encrypted Body as attachment\n")); 1446 DEBUG_INFO(("Adding Encrypted Body as attachment\n"));
1442 attach = (pst_item_attach*) pst_malloc(sizeof(pst_item_attach)); 1447 attach = (pst_item_attach*) pst_malloc(sizeof(pst_item_attach));
1443 memset(attach, 0, sizeof(pst_item_attach)); 1448 memset(attach, 0, sizeof(pst_item_attach));
1444 attach->next = item->attach; 1449 attach->next = item->attach;
1445 item->attach = attach; 1450 item->attach = attach;
1446 attach->data.data = item->email->encrypted_body.data; 1451 attach->data.data = item->email->encrypted_body.data;
1448 item->email->encrypted_body.data = NULL; 1453 item->email->encrypted_body.data = NULL;
1449 } 1454 }
1450 1455
1451 if (item->email->encrypted_htmlbody.data) { 1456 if (item->email->encrypted_htmlbody.data) {
1452 pst_item_attach* attach = (pst_item_attach*)pst_malloc(sizeof(pst_item_attach)); 1457 pst_item_attach* attach = (pst_item_attach*)pst_malloc(sizeof(pst_item_attach));
1453 DEBUG_EMAIL(("Adding encrypted HTML body as attachment\n")); 1458 DEBUG_INFO(("Adding encrypted HTML body as attachment\n"));
1454 attach = (pst_item_attach*) pst_malloc(sizeof(pst_item_attach)); 1459 attach = (pst_item_attach*) pst_malloc(sizeof(pst_item_attach));
1455 memset(attach, 0, sizeof(pst_item_attach)); 1460 memset(attach, 0, sizeof(pst_item_attach));
1456 attach->next = item->attach; 1461 attach->next = item->attach;
1457 item->attach = attach; 1462 item->attach = attach;
1458 attach->data.data = item->email->encrypted_htmlbody.data; 1463 attach->data.data = item->email->encrypted_htmlbody.data;
1472 int attach_num = 0; 1477 int attach_num = 0;
1473 for (attach = item->attach; attach; attach = attach->next) { 1478 for (attach = item->attach; attach; attach = attach->next) {
1474 pst_convert_utf8_null(item, &attach->filename1); 1479 pst_convert_utf8_null(item, &attach->filename1);
1475 pst_convert_utf8_null(item, &attach->filename2); 1480 pst_convert_utf8_null(item, &attach->filename2);
1476 pst_convert_utf8_null(item, &attach->mimetype); 1481 pst_convert_utf8_null(item, &attach->mimetype);
1477 DEBUG_EMAIL(("Attempting Attachment encoding\n")); 1482 DEBUG_INFO(("Attempting Attachment encoding\n"));
1478 if (!attach->data.data && attach->mimetype.str && !strcmp(attach->mimetype.str, RFC822)) { 1483 if (!attach->data.data && attach->mimetype.str && !strcmp(attach->mimetype.str, RFC822)) {
1479 DEBUG_EMAIL(("seem to have special embedded message attachment\n")); 1484 DEBUG_INFO(("seem to have special embedded message attachment\n"));
1480 find_rfc822_headers(extra_mime_headers); 1485 find_rfc822_headers(extra_mime_headers);
1481 write_embedded_message(f_output, attach, boundary, pst, extra_mime_headers); 1486 write_embedded_message(f_output, attach, boundary, pst, extra_mime_headers);
1482 } 1487 }
1483 else if (attach->data.data || attach->i_id) { 1488 else if (attach->data.data || attach->i_id) {
1484 if (mode == MODE_SEPARATE && !mode_MH) 1489 if (mode == MODE_SEPARATE && !mode_MH)
1810 char *temp = (char*) pst_malloc (strlen(f->name)+10); //enough room for 10 digits 1815 char *temp = (char*) pst_malloc (strlen(f->name)+10); //enough room for 10 digits
1811 1816
1812 sprintf(temp, "%s", f->name); 1817 sprintf(temp, "%s", f->name);
1813 check_filename(temp); 1818 check_filename(temp);
1814 while ((f->output = fopen(temp, "r"))) { 1819 while ((f->output = fopen(temp, "r"))) {
1815 DEBUG_MAIN(("need to increase filename because one already exists with that name\n")); 1820 DEBUG_INFO(("need to increase filename because one already exists with that name\n"));
1816 DEBUG_MAIN(("- increasing it to %s%d\n", f->name, x)); 1821 DEBUG_INFO(("- increasing it to %s%d\n", f->name, x));
1817 x++; 1822 x++;
1818 sprintf(temp, "%s%08d", f->name, x); 1823 sprintf(temp, "%s%08d", f->name, x);
1819 DEBUG_MAIN(("- trying \"%s\"\n", f->name)); 1824 DEBUG_INFO(("- trying \"%s\"\n", f->name));
1820 if (x == 99999999) { 1825 if (x == 99999999) {
1821 DIE(("create_enter_dir: Why can I not create a folder %s? I have tried %i extensions...\n", f->name, x)); 1826 DIE(("create_enter_dir: Why can I not create a folder %s? I have tried %i extensions...\n", f->name, x));
1822 } 1827 }
1823 fclose(f->output); 1828 fclose(f->output);
1824 } 1829 }
1828 } else { 1833 } else {
1829 free(temp); 1834 free(temp);
1830 } 1835 }
1831 } 1836 }
1832 1837
1833 DEBUG_MAIN(("f->name = %s\nitem->folder_name = %s\n", f->name, item->file_as.str)); 1838 DEBUG_INFO(("f->name = %s\nitem->folder_name = %s\n", f->name, item->file_as.str));
1834 if (mode != MODE_SEPARATE) { 1839 if (mode != MODE_SEPARATE) {
1835 check_filename(f->name); 1840 check_filename(f->name);
1836 if (!(f->output = fopen(f->name, "w"))) { 1841 if (!(f->output = fopen(f->name, "w"))) {
1837 DIE(("create_enter_dir: Could not open file \"%s\" for write\n", f->name)); 1842 DIE(("create_enter_dir: Could not open file \"%s\" for write\n", f->name));
1838 } 1843 }
1841 } 1846 }
1842 1847
1843 1848
1844 void close_enter_dir(struct file_ll *f) 1849 void close_enter_dir(struct file_ll *f)
1845 { 1850 {
1846 DEBUG_MAIN(("main: processed item count for folder %s is %i, skipped %i, total %i \n", 1851 DEBUG_INFO(("processed item count for folder %s is %i, skipped %i, total %i \n",
1847 f->dname, f->item_count, f->skip_count, f->stored_count)); 1852 f->dname, f->item_count, f->skip_count, f->stored_count));
1848 if (output_mode != OUTPUT_QUIET) printf("\t\"%s\" - %i items done, %i items skipped.\n", 1853 if (output_mode != OUTPUT_QUIET) {
1849 f->dname, f->item_count, f->skip_count); 1854 pst_debug_lock();
1855 printf("\t\"%s\" - %i items done, %i items skipped.\n", f->dname, f->item_count, f->skip_count);
1856 fflush(stdout);
1857 pst_debug_unlock();
1858 }
1850 if (f->output) { 1859 if (f->output) {
1851 struct stat st; 1860 struct stat st;
1852 fclose(f->output); 1861 fclose(f->output);
1853 stat(f->name, &st); 1862 stat(f->name, &st);
1854 if (!st.st_size) { 1863 if (!st.st_size) {