src/colorize.c

   1 #include <stdio.h>
   2 #include <string.h>
   3 #include <stdlib.h>
   4 #include <unistd.h>
   5 #include <getopt.h>
   6 #include <math.h>
   7 #include <time.h>
   8 #include "../lib/darray.h"
   9 #include "../lib/colorize.h"
  10 #include "ppm.h"
  11 #include "yuv.h"
  12 #include "mark.h"
  13 #include "dir_seperator.h"
  14 #ifdef WITH_OPENCV
  15 #include "opticalflow.h"
  16 #endif
  17
  18 #define min(x,y) ((x < y) ? x : y)
  19
  20 static void print_help(const char *app);
  21 static void print_test_help();
  22
  23 //#defbug SEQUENCE
  24 //#define SINGLE_K_TEST 2
  25
  26 /*
  27  * load sequence
  28  */
  29
  30 struct sequence {
  31         char filename[256];
  32 };
  33 static struct sequence *sequence = NULL;
  34
  35 /* load sequence list and return number of images in sequence */
  36 static int load_sequence(int *start, int *next, char *first_filename)
  37 {
  38         const char *name = "sequence";
  39         int count;
  40         int i, j, keyframe_before_start, eof;
  41         FILE *fp;
  42         char buffer[256], *p;
  43
  44         /* free previously used sequence */
  45         if (sequence)
  46                 free(sequence);
  47         sequence = NULL;
  48
  49 again:
  50         /* get number of frames until next keyframe or end of squence */
  51         fp = fopen(name, "r");
  52         if (!fp) {
  53                 printf("failed to load sequence '%s'\n", name);
  54                 return -1;
  55         }
  56         count = 0;
  57         j = 0;
  58         eof = 1;
  59         keyframe_before_start = 0;
  60         while(fgets(buffer, sizeof(buffer), fp)) {
  61                 p = strchr(buffer + 1, '\"');
  62                 if (!p)
  63                         break;
  64                 *p = '\0';
  65                 if (j == 0)
  66                         strcpy(first_filename, buffer + 1);
  67                 p++;
  68                 if ((*start) && j == (*start) - 1) {
  69                         if (strstr(p, "keyframe"))
  70                                 keyframe_before_start = 1;
  71                 }
  72                 if (j < (*start)) {
  73                         j++;
  74                         continue;
  75                 }
  76 #ifdef DEBUG_SEQUENCE
  77                 printf("counting %d\n", j);
  78 #endif
  79                 j++;
  80                 count++;
  81                 /* skip start frame, since it is always a keyframe */
  82                 if (count == 1)
  83                         continue;
  84                 if (strstr(p, "keyframe")) {
  85                         eof = 0;
  86                         break;
  87                 }
  88         }
  89         fclose(fp);
  90
  91         /* end of file case */
  92         if (count == 0) {
  93 #ifdef DEBUG_SEQUENCE
  94                 puts("end of file");
  95 #endif
  96                 *next = -1;
  97                 return 0;
  98         } else
  99         if (count == 1) {
 100 #ifdef DEBUG_SEQUENCE
 101                 puts("count=1");
 102 #endif
 103                 if (eof) {
 104 #ifdef DEBUG_SEQUENCE
 105                         puts("end of file at last frame");
 106 #endif
 107                         *next = -1;
 108                         return 0;
 109                 }
 110                 /* we are done, because we already had the last sequence */
 111                 if (!keyframe_before_start) {
 112 #ifdef DEBUG_SEQUENCE
 113                         puts("no keyframe before start");
 114 #endif
 115                         (*next) = -1;
 116                         return 0;
 117                 }
 118 #ifdef DEBUG_SEQUENCE
 119                 puts("keyframe before start");
 120 #endif
 121                 /* we just have a single last frame */
 122                 count = 1;
 123                 (*next) = (*start) + 1;
 124         } else
 125         /* if we have two keyframes side by side, we ignore it */
 126         if (count == 2 && !eof) {
 127 #ifdef DEBUG_SEQUENCE
 128                 puts("count=2");
 129 #endif
 130                 /* only if the frame before start is not a keyframe
 131                  * and we are not the first sequence, because that would mean
 132                  * that the first frame is a single sequence */
 133                 if (!keyframe_before_start && (*start) > 0) {
 134 #ifdef DEBUG_SEQUENCE
 135                         puts("no keyframe before start");
 136 #endif
 137                         (*start)++;
 138                         goto again;
 139                 }
 140 #ifdef DEBUG_SEQUENCE
 141                 puts("keyframe before start or first frame");
 142 #endif
 143                 /* now the start frame has one keyframe before and one after */
 144                 count = 1;
 145                 (*next) = (*start) + 1;
 146         } else {
 147                 (*next) = (*start) + count - 1;
 148         }
 149
 150         /* alloc memory for sequence and fill with data */
 151         sequence = (struct sequence *)malloc(sizeof(struct sequence) * count);
 152         if (!sequence) {
 153                 printf("no memory for sequence\n");
 154                 return -1;
 155         }
 156         memset((struct sequence *)sequence, 0, sizeof(struct sequence) * count);
 157         fp = fopen(name, "r");
 158         if (!fp) {
 159                 printf("failed to load sequence '%s'\n", name);
 160                 return -1;
 161         }
 162         i = 0;
 163         j = 0;
 164         while(fgets(buffer, sizeof(buffer), fp)) {
 165                 p = strchr(buffer + 1, '\"');
 166                 if (!p)
 167                         break;
 168                 if (j < (*start)) {
 169                         j++;
 170                         continue;
 171                 }
 172                 *p = '\0';
 173                 strcpy(sequence[i].filename, buffer + 1);
 174                 i++;
 175                 if (i == count)
 176                         break;
 177         }
 178         fclose(fp);
 179
 180         return count;
 181 }
 182
 183 /*
 184  * scaling
 185  */
 186
 187 /* scale down image in img_buffer by calculating average */
 188 static void scale_img(unsigned char *img_buffer, int width, int height, int scale)
 189 {
 190         int w, h, i, j, x, y;
 191         int r, g, b;
 192
 193         if (scale == 1)
 194                 return;
 195
 196         w = width / scale;
 197         h = height / scale;
 198
 199         for (i = 0; i < h; i++) {
 200                 for (j = 0; j < w; j++) {
 201                         r = g = b = 0;
 202                         for (y = 0; y < scale; y++) {
 203                                 for (x = 0; x < scale; x++) {
 204                                         r += img_buffer[((i*scale+y) * width + j*scale+x) * 3 + 0];
 205                                         g += img_buffer[((i*scale+y) * width + j*scale+x) * 3 + 1];
 206                                         b += img_buffer[((i*scale+y) * width + j*scale+x) * 3 + 2];
 207                                 }
 208                         }
 209                         img_buffer[(i * w + j)*3 + 0] = r / scale / scale;
 210                         img_buffer[(i * w + j)*3 + 1] = g / scale / scale;
 211                         img_buffer[(i * w + j)*3 + 2] = b / scale / scale;
 212                 }
 213         }
 214 }
 215
 216 /* scale down mark map in mark_buffer */
 217 static void scale_mark(unsigned char *mark_buffer, int width, int height, int scale)
 218 {
 219         int w, h, i, j, x, y;
 220         unsigned char c, temp;
 221
 222         if (scale == 1)
 223                 return;
 224
 225         w = width / scale;
 226         h = height / scale;
 227
 228         for (i = 0; i < h; i++) {
 229                 for (j = 0; j < w; j++) {
 230                         c = 0;
 231                         /* always use one index other than 0, if there is any in an areaa to be shrinked */
 232                         for (y = 0; y < scale; y++) {
 233                                 for (x = 0; x < scale; x++) {
 234                                         temp = mark_buffer[(i*scale+y) * width + j*scale+x];
 235                                         if (temp)
 236                                                 c = temp;
 237                                 }
 238                         }
 239                         mark_buffer[i * w + j] = c;
 240                 }
 241         }
 242 }
 243
 244 /*
 245  * options
 246  */
 247
 248 static int in_itr_num = 5, out_itr_num = 1, optical_flow = 1, bright_contrast = 1;
 249 int scale = 1, scalexyz = 999;
 250
 251 static enum test {
 252         NO_TEST = 0,
 253         FLOW_NEXT,
 254         FLOW_PREV,
 255         MARKED,
 256         MASK,
 257         MASK_COLOR,
 258         BC_ONLY,
 259         BC_IMAGE,
 260 } test = NO_TEST;
 261
 262 static int parse_test(const char *arg)
 263 {
 264         if (!strcmp(arg, "help")) {
 265                 print_test_help();
 266                 exit(0);
 267         }
 268 #ifdef WITH_OPENCV
 269         if (!strcmp(arg, "flow-next"))
 270                 return FLOW_NEXT;
 271         if (!strcmp(arg, "flow-prev"))
 272                 return FLOW_PREV;
 273 #endif
 274         if (!strcmp(arg, "marked"))
 275                 return MARKED;
 276         if (!strcmp(arg, "mask"))
 277                 return MASK;
 278         if (!strcmp(arg, "mask+color"))
 279                 return MASK_COLOR;
 280         if (!strcmp(arg, "bc-only"))
 281                 return BC_ONLY;
 282         if (!strcmp(arg, "bc-image"))
 283                 return BC_IMAGE;
 284
 285         return NO_TEST;
 286 }
 287
 288 static int handle_options(int argc, char **argv)
 289 {
 290         int skip_args = 0;
 291
 292         while (1) {
 293                 int option_index = 0, c;
 294                 static struct option long_options[] = {
 295                         {"help", 0, 0, 'h'},
 296                         {"in-itr-num", 1, 0, 'i'},
 297                         {"out-itr-num", 1, 0, 'o'},
 298                         {"zscale", 1, 0, 'z'},
 299                         {"brightness-contrast", 1, 0, 'b'},
 300                         {"optical-flow", 1, 0, 'f'},
 301                         {"scale", 1, 0, 's'},
 302                         {"test", 1, 0, 't'},
 303                         {0, 0, 0, 0},
 304                 };
 305
 306                 c = getopt_long(argc, argv, "hi:o:z:b:f:s:t:", long_options, &option_index);
 307
 308                 if (c == -1)
 309                         break;
 310
 311                 switch (c) {
 312                 case 'h':
 313                         print_help(argv[0]);
 314                         exit(0);
 315                 case 'i':
 316                         in_itr_num = atoi(optarg);
 317                         skip_args += 2;
 318                         break;
 319                 case 'o':
 320                         out_itr_num = atoi(optarg);
 321                         skip_args += 2;
 322                         break;
 323                 case 'z':
 324                         scalexyz = atoi(optarg);
 325                         skip_args += 2;
 326                         break;
 327                 case 'b':
 328                         bright_contrast = atoi(optarg);
 329                         skip_args += 2;
 330                         break;
 331                 case 'f':
 332                         optical_flow = atoi(optarg);
 333                         skip_args += 2;
 334                         break;
 335                 case 's':
 336                         scale = atoi(optarg);
 337                         skip_args += 2;
 338                         break;
 339                 case 't':
 340                         test = parse_test(optarg);
 341                         if (!test) {
 342                                 fprintf(stderr, "Invalid test '%s', use '--test help' to get a list of tests\n", optarg);
 343                                 exit(-1);
 344                         }
 345                         skip_args += 2;
 346                         break;
 347                 default:
 348                         break;
 349                 }
 350         }
 351
 352         return skip_args;
 353 }
 354
 355 /*
 356  * usage
 357  */
 358
 359 static void print_help(const char *app)
 360 {
 361         printf("Colorize version %s\n\n",
 362 #include "../version.h"
 363         );
 364         printf("Usage: %s [options] <grey ppm image> <marked ppm image> <result ppm image> [<frames> <start>]\n", app);
 365         printf("       Colorize grey image using maked image and save to result image.\n");
 366         printf("       If frames and start frame is given, image names must include printf integer formatting (e.g. %%04d).\n");
 367         printf("Usage: %s [options] <grey ppm image> marked <result ppm image>\n", app);
 368         printf("       Colorize grey image using marked mask + palette and save to result image.\n");
 369         printf("Usage: %s [options] sequence [list | <start with frame> [<stop with frame>]]\n", app);
 370         printf("       Colorize movie sequence (generated by colorize gui) as found in the current directory.\n");
 371         printf("       Use list to view sequence segments between keyframes.\n");
 372         printf("\nOptions:\n");
 373         printf(" -h --help                           This help\n");
 374         printf(" -i --in-itr-num <num>               Alter inner iterations (weightening count) of colorization algorithm (default=%d)\n", in_itr_num);
 375         printf(" -o --out-itr-num <num>              Alter outer iterations (complete turns) of colorization algorithm (default=%d)\n", out_itr_num);
 376         printf(" -z --zscale <levels>                How many grids (staring with the finest) should be scaled in z direction to generate the next coarse grid ");
 377         if (scalexyz < 999)
 378                 printf("(default=%d)\n", scalexyz);
 379         else
 380                 printf("(default=infinite)\n");
 381         printf(" -b --brightness-contrast [0 | 1]    Apply brightnes and contrast, if defined in palette by GUI (default=%d)\n", bright_contrast);
 382 #ifdef WITH_OPENCV
 383         printf(" -f --optical-flow [0 | 1]           Apply optical flow, if defined by GUI (default=%d)\n", optical_flow);
 384 #endif
 385         printf(" -s --scale [1..n]                   Scale down by the given factor for quick and dirty previews (default=%d)\n", scale);
 386         printf(" -t --test <test>                    Generate test images. Use 'help' for list of tests\n");
 387 }
 388
 389 static void print_test_help()
 390 {
 391         printf(" -t --test <test>        Generate test images...\n");
 392 #ifdef WITH_OPENCV
 393         printf("           flow-next     Optical flow plane to next image\n");
 394         printf("           flow-prev     Optical flow plane to previous image\n");
 395 #endif
 396         printf("           marked        Only apply makred colors to grey image\n");
 397         printf("           mask          Show mask of marked areas\n");
 398         printf("           mask+color    Show mask of marked areas + color\n");
 399         printf("           bc-only       Only apply brightness+contrast, leave colors of grey image as is\n");
 400         printf("           bc-image      Show brightness+contrast change on grey image as uv components\n");
 401 }
 402
 403 /*
 404  * main function
 405  */
 406
 407 int main(int argc, char *argv[])
 408 {
 409         darray_t *gI = NULL, *cI = NULL, *markIm = NULL, *ntscIm = NULL, *resultIm;
 410         darray_t *dx = NULL, *dy = NULL, *idx = NULL, *idy = NULL;
 411 #ifdef WITH_OPENCV
 412         int *flow_map_x = NULL, *flow_map_y = NULL;
 413 #endif
 414         double *ptr, *ptr2, *ptr3;
 415         double diff, sum;
 416         int dims[4];
 417         int width = 0, height = 0, w, h, k = 1, index = 0, count;
 418         unsigned char *img_buffer, *img_buffer_all = NULL, *mark_buffer = NULL;
 419         int rc, i, ii, j;
 420         time_t start, end;
 421         const char* filename;
 422         char first_filename[256];
 423         int seq_offset = 0, seq_next = 0;
 424         int features, change_bc;
 425         int skip_args;
 426
 427         skip_args = handle_options(argc, argv);
 428         argc -= skip_args;
 429         argv += skip_args;
 430
 431         if (argc <= 1) {
 432                 print_help(argv[0]);
 433                 return 0;
 434         }
 435
 436 next_sequence:
 437         if (argc > 1 && !strcmp(argv[1], "sequence")) {
 438                 k = load_sequence(&seq_offset, &seq_next, first_filename);
 439                 if (k == 0)
 440                         return 0;
 441                 printf("Got %d frames from sequence (frames %d..%d)\n", k, seq_offset, seq_offset + k - 1);
 442                 if (argc > 2 && (!strcmp(argv[2], "list") || atoi(argv[2]) > seq_offset)) {
 443                         seq_offset = seq_next;
 444                         goto next_sequence;
 445                 }
 446         } else if (argc <= 3) {
 447                 print_help(argv[0]);
 448                 return 0;
 449         } else if (argc > 5) {
 450                 k = atoi(argv[4]);
 451                 index = atoi(argv[5]);
 452         }
 453         if (k <= 0)
 454                 return 0;
 455
 456 #ifdef SINGLE_K_TEST
 457         if (k > SINGLE_K_TEST)
 458                 k = SINGLE_K_TEST;
 459 #endif
 460
 461         time(&start);
 462
 463         change_bc = 0;
 464         for (count = 0; count < k; count++) {
 465                 if (sequence) {
 466                         filename = sequence[count].filename;
 467                         /* first_filename is set by load_sequence */
 468                 } else {
 469                         filename = argv[1];
 470                         strcpy(first_filename, argv[1]);
 471                 }
 472 #ifdef WITH_OPENCV
 473                 // load flow settings
 474                 if (sequence || !strcmp(argv[2], "marked")) {
 475                         flow_default();
 476                         if (optical_flow)
 477                                 load_flow(first_filename);
 478                         if (flow_enable == 0 && (test == FLOW_NEXT || test == FLOW_PREV)) {
 479                                 fprintf(stderr, "Cannot test optical flow, because it is not enabled by GUI.\n");
 480                                 exit (0);
 481                         }
 482                 }
 483 #endif
 484                 // load original image and convert their RGB components to double RGB array
 485                 rc = load_img(-1, &img_buffer, &w, &h, filename, index + count);
 486                 if (rc) {
 487                         fprintf(stderr, "Failed to load grey image\n");
 488                         return 0;
 489                 }
 490                 scale_img(img_buffer, w, h, scale);
 491                 if (count == 0) {
 492                         width = w / scale;
 493                         height = h / scale;
 494                 }
 495                 if (w/scale != width || h/scale != height) {
 496                         fprintf(stderr, "Error: All input images must have equal dimenstions.\n");
 497                         return 0;
 498                 }
 499                 // now we know the dimensions, so we can create input arrays
 500                 if (!gI) {
 501                         dims[0] = width; dims[1] = height; dims[2] = 3; dims[3] = k;
 502                         gI = darrayCreate(4, dims);
 503                 }
 504                 if (!gI) {
 505                         printf("failed to create grey image array\n");
 506                         exit (0);
 507                 }
 508                 if (!cI) {
 509                         dims[0] = width; dims[1] = height; dims[2] = 3; dims[3] = k;
 510                         cI = darrayCreate(4, dims);
 511                 }
 512                 if (!cI) {
 513                         printf("failed to create marked image array\n");
 514                         exit (0);
 515                 }
 516                 ptr = darrayGetPr(gI) + width*height*3*count;
 517                 img2array(img_buffer, width, height, ptr, width, height);
 518 #ifdef WITH_OPENCV
 519                 if (k > 1 && flow_enable) {
 520                         if (!img_buffer_all)
 521                                 img_buffer_all = malloc(width*height*3*k);
 522                         if (!img_buffer_all) {
 523                                 printf("failed to create grey image array\n");
 524                                 exit (0);
 525                         }
 526                         memcpy(img_buffer_all + width*height*3*count, img_buffer, width*height*3);
 527                 }
 528 #endif
 529                 free(img_buffer);
 530
 531                 if (sequence || !strcmp(argv[2], "marked")) {
 532                         char name[256];
 533                         unsigned char c;
 534                         // load marked mask and convert their RGB components to double YUV array
 535                         memcpy(darrayGetPr(cI) + width*height*3*count, darrayGetPr(gI) + width*height*3*count, width*height*3 * sizeof(double));
 536                         /* add extra memory for unscaled data to prevent buffer overflow */
 537                         if (!mark_buffer)
 538                                 mark_buffer = (unsigned char *)malloc(width*height*k + w*h);
 539                         if (!mark_buffer) {
 540                                 printf("no memory for mark buffer\n");
 541                                 exit (0);
 542                         }
 543                         if (load_palette(first_filename)) {
 544                                 printf("failed to load palette for file: '%s'\n", filename);
 545                                 exit (0);
 546                         }
 547                         sprintf(name, filename, index + count);
 548                         /* always load full unscaled image, then scale down */
 549                         if (load_marked(mark_buffer + width*height*count, w, h, name) == 0) {
 550                                 scale_mark(mark_buffer + width*height*count, w, h, scale);
 551                                 ptr = darrayGetPr(cI) + width*height*3*count;
 552                                 for (i = 0; i < height; i++) {
 553                                         for (j = 0; j < width; j++) {
 554                                                 /* do not apply mask on index 0 */
 555                                                 c = mark_buffer[i*width+j + width*height*count];
 556                                                 if (c == 0)
 557                                                         continue;
 558                                                 /* check for any brightness/contrast change */
 559                                                 if (bright_contrast && (mark_palette[c-1].bright != 0 || mark_palette[c-1].contrast != 1))
 560                                                         change_bc = 1;
 561                                                 /* do not apply white pixles, this meas: keep original color */
 562                                                 if (mark_palette[c-1].r == 255 && mark_palette[c-1].g == 255 && mark_palette[c-1].b == 255)
 563                                                         continue;
 564                                                 ptr[i*width+j] = mark_palette[c-1].r / 255.0F;
 565                                                 ptr[i*width+j + width*height] = mark_palette[c-1].g / 255.0F;
 566                                                 ptr[i*width+j + width*height*2] = mark_palette[c-1].b / 255.0F;
 567                                         }
 568                                 }
 569                         } else
 570                                 memset(mark_buffer + width*height*count, 0, width*height);
 571                 } else {
 572                         // load marked image and convert their RGB components to double YUV array
 573                         rc = load_img(-1, &img_buffer, &w, &h, argv[2], index + count);
 574                         if (!rc) {
 575                                 if (w/scale != width || h/scale != height) {
 576                                         fprintf(stderr, "Error: All input images must have equal dimenstions.\n");
 577                                         return 0;
 578                                 }
 579                                 ptr = darrayGetPr(cI) + width*height*3*count;
 580                                 img2array(img_buffer, width, height, ptr, width, height);
 581                                 free(img_buffer);
 582                         } else {
 583                                 fprintf(stderr, "Failed to load marked image, omitting...\n");
 584                                 memcpy(darrayGetPr(cI) + width*height*3*count, darrayGetPr(gI) + width*height*3*count, width*height*3 * sizeof(double));
 585                         }
 586                 }
 587         }
 588
 589         /* create color mask and ntsc arrays for the colorization process */
 590         dims[0] = width; dims[1] = height; dims[2] = k;
 591         markIm = darrayCreate(3, dims);
 592         if (!markIm) {
 593                 printf("failed to create mark array\n");
 594                 exit (0);
 595         }
 596         features = (change_bc) ? 4 : 2;
 597         dims[0] = width; dims[1] = height; dims[2] = features+1; dims[3] = k;
 598         ntscIm = darrayCreate(4, dims);
 599         if (!ntscIm) {
 600                 printf("failed to create ntsc array\n");
 601                 exit (0);
 602         }
 603
 604         for (count = 0; count < k; count++) {
 605                 if (sequence || !strcmp(argv[2], "marked")) {
 606                         unsigned char c;
 607                         ptr = darrayGetPr(markIm) + width*height*count;
 608                         // use marked mask to fill markIm
 609                         for (i = 0; i < height; i++) {
 610                                 for (j = 0; j < width; j++) {
 611                                         if (j < width && i < height) {
 612                                                 /* do not apply mask on index 0 */
 613                                                 c = mark_buffer[i*width+j + width*height*count];
 614                                         } else
 615                                                 c = 0;
 616                                         if (c)
 617                                                 ptr[i*width+j] = 1.0F;
 618                                         else
 619                                                 ptr[i*width+j] = 0.0F;
 620                                 }
 621                         }
 622                 } else {
 623                         // fill color image with marked pixles
 624                         // - calculate the difference between two images (original image - color image)
 625                         // - convert into absolute (positive values)
 626                         // - sum all components to get grey image
 627                         // - apply threshold (pixle is 1F, if the absolute difference is > 0.01F)
 628                         // original code: markIm=(sum(abs(gI-cI),3)>0.01);
 629                         ptr = darrayGetPr(gI) + width*height*3*count;
 630                         ptr2 = darrayGetPr(cI) + width*height*3*count;
 631                         ptr3 = darrayGetPr(markIm) + width*height*count;
 632                         for (i = 0, ii = width * height; i < ii; i++) {
 633                                 diff = 0;
 634                                 sum = ptr[i] - ptr2[i];
 635                                 if (sum < 0)
 636                                         diff -= sum;
 637                                 else
 638                                         diff += sum;
 639                                 sum = ptr[i + ii] - ptr2[i + ii];
 640                                 if (sum < 0)
 641                                         diff -= sum;
 642                                 else
 643                                         diff += sum;
 644                                 sum = ptr[i + ii + ii] - ptr2[i + ii + ii];
 645                                 if (sum < 0)
 646                                         diff -= sum;
 647                                 else
 648                                         diff += sum;
 649                                 if (diff > 0.01)
 650                                         ptr3[i] = 1.0F;
 651                                 else
 652                                         ptr3[i] = 0.0F;
 653                         }
 654                 }
 655
 656                 // convert original image into YUV
 657                 ptr = darrayGetPr(gI) + width*height*3*count;
 658                 rgb2yuv(ptr, ptr, width, height);
 659
 660                 // convert maked image into YUV
 661                 ptr = darrayGetPr(cI) + width*height*3*count;
 662                 rgb2yuv(ptr, ptr, width, height);
 663
 664                 if (test != BC_ONLY) {
 665                         if (sequence || !strcmp(argv[2], "marked")) {
 666                                 unsigned char c;
 667                                 // generate NTSC image: use luminance from original image and chrominance from original or marked image
 668                                 ptr = darrayGetPr(gI) + width*height*3*count;
 669                                 ptr2 = darrayGetPr(cI) + width*height*3*count;
 670                                 ptr3 = darrayGetPr(ntscIm) + width*height*(features+1)*count;
 671                                 /* use original y component */
 672                                 memcpy(ptr3, ptr, width * height * sizeof(double));
 673                                 /* apply new uv components */
 674                                 for (i = 0; i < height; i++) {
 675                                         for (j = 0; j < width; j++) {
 676                                                 c = mark_buffer[i*width+j + width*height*count];
 677                                                 if (c == 0) {
 678                                                         ptr3[width * height + width * i + j] = ptr[width * height + width * i + j];
 679                                                         ptr3[width * height * 2 + width * i + j] = ptr[width * height * 2 + width * i + j];
 680                                                 } else {
 681                                                         ptr3[width * height + width * i + j] = ptr2[width * height + width * i + j];
 682                                                         ptr3[width * height * 2 + width * i + j] = ptr2[width * height * 2 + width * i + j];
 683                                                 }
 684                                         }
 685                                 }
 686                         } else {
 687                                 // generate NTSC image: use luminance from original image and chrominance from maked image
 688                                 ptr = darrayGetPr(gI) + width*height*3*count;
 689                                 ptr2 = darrayGetPr(cI) + width*height*3*count;
 690                                 ptr3 = darrayGetPr(ntscIm) + width*height*(features+1)*count;
 691                                 memcpy(ptr3, ptr, width * height * sizeof(double));
 692                                 memcpy(ptr3 + width * height, ptr2 + width * height, width * height * sizeof(double));
 693                                 memcpy(ptr3 + width * height * 2, ptr2 + width * height * 2, width * height * sizeof(double));
 694                         }
 695                 } else {
 696                         /* use grey image as result if BC_ONLY test is selected */
 697                         ptr = darrayGetPr(gI) + width*height*3*count;
 698                         ptr2 = darrayGetPr(ntscIm) + width*height*(features+1)*count;
 699                         memcpy(ptr2, ptr, width * height * 3 * sizeof(double));
 700                 }
 701
 702                 if (change_bc) {
 703                         unsigned char c;
 704                         ptr2 = darrayGetPr(ntscIm) + width*height*(features+1)*count;
 705                         /* use original y component */
 706                         memcpy(ptr2, ptr, width * height * sizeof(double));
 707                         /* apply brightness and contrast from makred pixles to uv components of grey image */
 708                         for (i = 0; i < height; i++) {
 709                                 for (j = 0; j < width; j++) {
 710                                         /* use unchanged brightness and contrast on index 0 */
 711                                         c = mark_buffer[i*width+j + width*height*count];
 712                                         if (c == 0) {
 713                                                 ptr2[i*width+j + width*height*3] = 0;
 714                                                 ptr2[i*width+j + width*height*4] = 0.1;
 715                                         } else {
 716                                                 ptr2[i*width+j + width*height*3] = mark_palette[c-1].bright / 10.0;
 717                                                 ptr2[i*width+j + width*height*4] = mark_palette[c-1].contrast / 10.0;
 718                                         }
 719                                 }
 720                         }
 721                 }
 722         }
 723
 724         /* create flow vectors */
 725         dims[0] = width; dims[1] = height; dims[2] = k - 1;
 726         dx = darrayCreate(3, dims);
 727         if (!dx) {
 728                 printf("failed to create array\n");
 729                 exit (0);
 730         }
 731         dy = darrayCreate(3, dims);
 732         if (!dy) {
 733                 printf("failed to create array\n");
 734                 exit (0);
 735         }
 736         idx = darrayCreate(3, dims);
 737         if (!idx) {
 738                 printf("failed to create array\n");
 739                 exit (0);
 740         }
 741         idy = darrayCreate(3, dims);
 742         if (!idy) {
 743                 printf("failed to create array\n");
 744                 exit (0);
 745         }
 746 #ifdef WITH_OPENCV
 747         if (k > 1 && flow_enable) {
 748                 printf("Calculating optical flow for %d frames: steps=%d win=%d max=%d\n", k, flow_steps/scale, flow_win/scale, flow_max/scale);
 749                 flow_map_x = (int *)malloc(sizeof(int)*width*height);
 750                 if (!flow_map_x) {
 751                         printf("failed to alloc array\n");
 752                         exit (0);
 753                 }
 754                 flow_map_y = (int *)malloc(sizeof(int)*width*height);
 755                 if (!flow_map_y) {
 756                         printf("failed to alloc array\n");
 757                         exit (0);
 758                 }
 759         }
 760         for (count = 0; count < k-1; count++) {
 761                 if (flow_map_x && flow_map_y)
 762                         create_flow_maps(NULL, img_buffer_all + width*height*3*(count+1), img_buffer_all + width*height*3*count, width, height, flow_steps/scale, flow_win/scale, flow_max/scale, NULL, NULL, flow_map_x, flow_map_y, NULL);
 763                 ptr = darrayGetPr(dx) + width*height*count;
 764                 if (flow_map_x) {
 765                         for (j = 0; j < height; j++) {
 766                                 for (i = 0; i < width; i++)
 767                                         ptr[i + width * j] = flow_map_x[width*j + i];
 768                         }
 769                 }
 770                 ptr = darrayGetPr(dy) + width*height*count;
 771                 if (flow_map_y) {
 772                         for (j = 0; j < height; j++) {
 773                                 for (i = 0; i < width; i++)
 774                                         ptr[i + width * j] = flow_map_y[width*j + i];
 775                         }
 776                 }
 777                 if (flow_map_x && flow_map_y)
 778                         create_flow_maps(img_buffer_all + width*height*3*count, NULL, img_buffer_all + width*height*3*(count+1), width, height, flow_steps/scale, flow_win/scale, flow_max/scale, flow_map_x, flow_map_y, NULL, NULL, NULL);
 779                 ptr = darrayGetPr(idx) + width*height*count;
 780                 if (flow_map_x) {
 781                         for (j = 0; j < height; j++) {
 782                                 for (i = 0; i < width; i++)
 783                                         ptr[i + width * j] = flow_map_x[width*j + i];
 784                         }
 785                 }
 786                 ptr = darrayGetPr(idy) + width*height*count;
 787                 if (flow_map_y) {
 788                         for (j = 0; j < height; j++) {
 789                                 for (i = 0; i < width; i++)
 790                                         ptr[i + width * j] = flow_map_y[width*j + i];
 791                         }
 792                 }
 793         }
 794 #endif
 795         free(img_buffer_all);
 796         img_buffer_all = NULL;
 797
 798         darrayDestroy(gI);
 799         gI = NULL;
 800         darrayDestroy(cI);
 801         cI = NULL;
 802
 803         if (test != FLOW_NEXT && test != FLOW_PREV && test != MARKED && test != MASK && test != MASK_COLOR && test != BC_ONLY && test != BC_IMAGE) {
 804                 printf("Colorizing %d frames, please wait...\n", k);
 805                 resultIm = colorize(ntscIm, markIm, dx, dy, idx, idy, in_itr_num, out_itr_num, scalexyz);
 806                 if (!resultIm) {
 807                         if (k > 1)
 808                                 printf("No memory! Use smaller frames or less frames between key frames or add more memory.");
 809                         else
 810                                 printf("No memory! Use smaller image or add more memory.");
 811                         exit(-1);
 812                 }
 813         } else
 814                 resultIm = darrayClone(ntscIm);
 815
 816         /* if we have a change, we apply brightness+contrast from bcIm */
 817         if (change_bc) {
 818                 for (count = 0; count < k; count++) {
 819                         ptr = darrayGetPr(ntscIm) + width*height*(features+1)*count;
 820                         ptr2 = darrayGetPr(resultIm) + width*height*(features+1)*count;
 821                         for (i = 0; i < height; i++) {
 822                                 for (j = 0; j < width; j++) {
 823                                         /* apply contrast */
 824                                         ptr2[width * i + j] = (ptr2[width * i + j] - 0.5) * ptr[width * height * 4 + width * i + j] * 10.0 + 0.5;
 825                                         /* apply brightness */
 826                                         ptr2[width * i + j] += ptr[width * height * 3 + width * i + j] * 10.0;
 827                                         if (ptr2[width * i + j] < 0)
 828                                                 ptr2[width * i + j] = 0;
 829                                         if (ptr2[width * i + j] > 1)
 830                                                 ptr2[width * i + j] = 1;
 831 #if 0
 832 #warning TEST: show brightness and contrast change as uv vectors on a grey array */
 833 ptr2[width * i + j] = 0.5;
 834 ptr2[width * height + width * i + j] = ptr[width * height + width * i + j] * 10;
 835 ptr2[width * height * 2 + width * i + j] = ptr[width * height * 2 + width * i + j] * 10 - 1;
 836 #endif
 837                                 }
 838                         }
 839                 }
 840         }
 841
 842         darrayDestroy(ntscIm);
 843         ntscIm = NULL;
 844
 845 #ifdef WITH_OPENCV
 846         if (test == FLOW_NEXT || test == FLOW_PREV) {
 847                 /* apply flow planes to result image as u and y vector */
 848                 for (count = 0; count < k; count++) {
 849                         ptr = darrayGetPr(resultIm) + width*height*(features+1)*count;
 850                         if (test == FLOW_NEXT) {
 851                                 ptr2 = darrayGetPr(dx) + width*height*count;
 852                                 ptr3 = darrayGetPr(dy) + width*height*count;
 853                         } else {
 854                                 ptr2 = darrayGetPr(idx) + width*height*count;
 855                                 ptr3 = darrayGetPr(idy) + width*height*count;
 856                         }
 857                         for (i = 0; i < height; i++) {
 858                                 for (j = 0; j < width; j++) {
 859                                         ptr[width * i + j] = 0.5;
 860                                         if (count < k-1) {
 861                                                 ptr[width * i + j + width*height] = ptr2[width * i + j] / flow_max;
 862                                                 ptr[width * i + j + width*height*2] = ptr3[width * i + j] / flow_max;
 863                                         } else {
 864                                                 ptr[width * i + j + width*height] = 0;
 865                                                 ptr[width * i + j + width*height*2] = 0;
 866                                         }
 867                                 }
 868                         }
 869                 }
 870         }
 871 #endif
 872
 873         if (test == MASK) {
 874                 /* apply maked mask as image */
 875                 for (count = 0; count < k; count++) {
 876                         ptr = darrayGetPr(resultIm) + width*height*(features+1)*count;
 877                         ptr2 = darrayGetPr(markIm) + width*height*count;
 878                         for (i = 0; i < height; i++) {
 879                                 for (j = 0; j < width; j++) {
 880                                         ptr[width * i + j] = ptr2[width * i + j];
 881                                         ptr[width * i + j + width*height] = ptr2[width * i + j];
 882                                         ptr[width * i + j + width*height*2] = ptr2[width * i + j];
 883                                 }
 884                         }
 885                 }
 886         }
 887
 888         if (test == MASK_COLOR) {
 889                 /* apply maked mask on grey image */
 890                 for (count = 0; count < k; count++) {
 891                         ptr = darrayGetPr(resultIm) + width*height*(features+1)*count;
 892                         ptr2 = darrayGetPr(markIm) + width*height*count;
 893                         for (i = 0; i < height; i++) {
 894                                 for (j = 0; j < width; j++) {
 895                                         /* darken unmarked areas, make maked areas with uniformed brightness */
 896                                         if (ptr2[width * i + j] < 0.5F)
 897                                                 ptr[width * i + j] = ptr[width * i + j] / 4;
 898                                         else
 899                                                 ptr[width * i + j] = 0.5F;
 900                                 }
 901                         }
 902                 }
 903         }
 904
 905         if (test == BC_IMAGE) {
 906                 /* apply bc image as result image */
 907                 for (count = 0; count < k; count++) {
 908                         ptr = darrayGetPr(resultIm) + width*height*(features+1)*count;
 909                         ptr2 = darrayGetPr(resultIm) + width*height*(features+1)*count;
 910                         /* use uniformed brightness as y component */
 911                         for (i = 0; i < height; i++) {
 912                                 for (j = 0; j < width; j++) {
 913                                         ptr[width * i + j] = 0.5F;
 914                                 }
 915                         }
 916                         if (change_bc) {
 917                                 ptr2 = darrayGetPr(resultIm) + width*height*(features+1)*count;
 918                                 memcpy(ptr+width*height, ptr2+width*height*3, width * height * sizeof(double));
 919                                 memcpy(ptr+width*height*2, ptr2+width*height*4, width * height * sizeof(double));
 920                         } else {
 921                                 memset(ptr+width*height, 0, width * height * sizeof(double));
 922                                 memset(ptr+width*height*2, 0, width * height * sizeof(double));
 923                         }
 924                 }
 925         }
 926
 927         // save result YUV array to image with RGB components
 928         img_buffer = (unsigned char *)malloc(width * height * 3);
 929         if (!img_buffer) {
 930                 fprintf(stderr, "Failed to allocate image buffer\n");
 931                 return 0;
 932         }
 933         for (count = 0; count < k; count++) {
 934                 ptr = darrayGetPr(resultIm) + width*height*(features+1)*count;
 935                 yuv2rgb(ptr, ptr, width, height);
 936                 array2img(ptr, width, height, img_buffer, width, height);
 937                 if (sequence) {
 938                         static char name[256], *p, *q;
 939                         p = sequence[count].filename;
 940                         while((q = strchr(p, DIR_SEPERATOR)))
 941                                 p = q + 1;
 942                         strcpy(name, sequence[count].filename);
 943                         name[p - sequence[count].filename] = '\0';
 944                         strcat(name, "colorized_");
 945                         strcat(name, p);
 946                         filename = name;
 947                 } else
 948                         filename = argv[3];
 949                 save_img(img_buffer, width, height, filename, index + count);
 950         }
 951         free(img_buffer);
 952
 953         time(&end);
 954         printf("Elapsed time: %d minutes, %d seconds\n", (int)(end-start)/60, (int)(end-start)%60);
 955
 956         // destroy
 957         darrayDestroy(resultIm);
 958         resultIm = NULL;
 959         darrayDestroy(dx);
 960         dx = NULL;
 961         darrayDestroy(dy);
 962         dy = NULL;
 963         darrayDestroy(idx);
 964         idx = NULL;
 965         darrayDestroy(idy);
 966         idy = NULL;
 967 #ifdef WITH_OPENCV
 968         free(flow_map_x);
 969         flow_map_x = NULL;
 970         free(flow_map_y);
 971         flow_map_y = NULL;
 972 #endif
 973         darrayDestroy(markIm);
 974         markIm = NULL;
 975
 976         free(mark_buffer);
 977         mark_buffer = NULL;
 978
 979         darrayDone();
 980
 981 #ifdef SINGLE_K_TEST
 982         exit(0);
 983 #endif
 984         if (sequence) {
 985                 /* if end frame is not given or if not reached */
 986                 if (argc <= 3 || atoi(argv[3]) > seq_offset + k) {
 987                         seq_offset = seq_offset + k - 1;
 988                         goto next_sequence;
 989                 }
 990         }
 991
 992         return 0;
 993 }
 994