tools/rerate.c

   1 #include <stdio.h>
   2 #include <string.h>
   3 #include <stdlib.h>
   4 #include <unistd.h>
   5 #include <getopt.h>
   6 #include <math.h>
   7 #include "../src/img.h"
   8 #include <opencv2/core/core_c.h>
   9 #include <opencv2/imgproc/imgproc_c.h>
  10 #include <opencv2/video/tracking.hpp>
  11 #include<opencv2/highgui/highgui.hpp>
  12
  13 /* test single flow without smoothing */
  14 //#define TEST_SINGLE
  15
  16 /* test input image */
  17 //#define TEST_INPUT
  18
  19 int flow_window = 30, border = 0;
  20 double limit_frames = 0;
  21 int interlace = 0, swap = 0;
  22
  23 static void field_img(int width, int height, unsigned short *img, int odd)
  24 {
  25         int y, x;
  26         signed int value;
  27
  28         /* use even lines: 0, 2, 4... */
  29         /* use odd lines: 1, 3, 5... */
  30         for (y = (!!odd); y < height-2; y += 2) {
  31                 for (x = 0; x < width*3; x++) {
  32                         value = img[width * y * 3 + x];
  33                         value += img[width * (y+2) * 3 + x];
  34                         img[width * (y+1) * 3 + x] = (value >> 1);
  35                 }
  36         }
  37         /* duplicate border line that can't be interpolated */
  38         if (!odd)
  39                 memcpy(img + width * (height-1) * 3, img + width * (height-2) * 3, width * 3 * sizeof(unsigned short));
  40         else
  41                 memcpy(img, img + width * 3, width * 3 * sizeof(unsigned short));
  42 }
  43
  44 static inline void smooth_pixle(double *array, int width, int height, int i, int j)
  45 {
  46         double val = 0;
  47         int num = 0;
  48         /* left-top */
  49         if (i > 0 &&            j > 0) {
  50                 val += array[i*width+j-1-width];
  51                 num++;
  52         }
  53         /* top */
  54         if (i > 0) {
  55                 val += array[i*width+j-width];
  56                 num++;
  57         }
  58         /* right-top */
  59         if (i > 0 &&            j < width-1) {
  60                 val += array[i*width+j+1-width];
  61                 num++;
  62         }
  63         /* left */
  64         if (                    j > 0) {
  65                 val += array[i*width+j-1];
  66                 num++;
  67         }
  68         /* right */
  69         if (                    j < width-1) {
  70                 val += array[i*width+j+1];
  71                 num++;
  72         }
  73         /* left-bottom */
  74         if (i < height-1 &&     j > 0) {
  75                 val += array[i*width+j-1+width];
  76                 num++;
  77         }
  78         /* bottom */
  79         if (i < height-1) {
  80                 val += array[i*width+j+width];
  81                 num++;
  82         }
  83         /* right-bottom */
  84         if (i < height-1 &&     j < width-1) {
  85                 val += array[i*width+j+1+width];
  86                 num++;
  87         }
  88         if (num)
  89                 array[i*width+j] = val/num;
  90 }
  91
  92 /* interpolate missing pixles in array */
  93 static void smooth(double *array, double *mask, int width, int height)
  94 {
  95         int i, j, iter;
  96
  97         for (iter = 0; iter < 3; iter++) {
  98                 for (i = 0; i < height; i++) {
  99                         for (j = 0; j < width; j++) {
 100                                 if (*mask++ == 0.0)
 101                                         continue;
 102                                 smooth_pixle(array, width, height, i, j);
 103                         }
 104                 }
 105                 for (i = height-1; i >= 0; i--) {
 106                         for (j = width-1; j >= 0; j--) {
 107                                 if (*--mask == 0.0)
 108                                         continue;
 109                                 smooth_pixle(array, width, height, i, j);
 110                         }
 111                 }
 112         }
 113 }
 114
 115 /* use optical flow to interpolate two images with given offset */
 116 static int interpolate(unsigned short *img1, unsigned short *img2, unsigned short *res, int width, int height, double offset, int frame)
 117 {
 118         static int lastframe = -1;
 119         static CvMat *flow1 = NULL, *flow2 = NULL;
 120         static IplImage *image1, *image2;
 121         double *array1, *array2;
 122         CvSize size;
 123         int i, j, x, y;
 124         unsigned short *i1, *i2;
 125         unsigned int grey;
 126         double alpha;
 127         const float *f;
 128         const float pyrScale = 0.5;
 129         const float levels = 3;
 130         const float winsize = flow_window;
 131         const float iterations = 8;
 132         const float polyN = 5;
 133         const float polySigma = 1.2;
 134         const int flags = 0;
 135
 136         if (lastframe != frame && flow1) {
 137                 cvReleaseMat(&flow1);
 138                 flow1 = NULL;
 139         }
 140         if (lastframe != frame && flow2) {
 141                 cvReleaseMat(&flow2);
 142                 flow2 = NULL;
 143         }
 144         lastframe = frame;
 145
 146         array1 = calloc(width*height*5*sizeof(*array1), 1);
 147         array2 = calloc(width*height*5*sizeof(*array2), 1);
 148         if (!array1 || !array2) {
 149                 printf("Failed to create image array\n");
 150                 return -1;
 151         }
 152
 153         /* if we need to make flow maps, we do it */
 154         if (!flow1 || !flow2) {
 155                 printf(" -> calculating flow %d<->%d\n", frame, frame+1);
 156                 flow1 = cvCreateMat(height, width, CV_32FC2);
 157                 flow2 = cvCreateMat(height, width, CV_32FC2);
 158                 if (!flow1 || !flow2) {
 159                         printf("Failed to create flow map\n");
 160                         return -1;
 161                 }
 162
 163                 size.width = width;
 164                 size.height = height;
 165 //              image1 = cvCreateImage(size, IPL_DEPTH_32F, 1);
 166 //              image2 = cvCreateImage(size, IPL_DEPTH_32F, 1);
 167                 image1 = cvCreateImage(size, IPL_DEPTH_16U, 1);
 168                 image2 = cvCreateImage(size, IPL_DEPTH_16U, 1);
 169                 if (!image1 || !image2) {
 170                         printf("Failed to create image for optical flow\n");
 171                         return -1;
 172                 }
 173                 i1 = img1;
 174                 i2 = img2;
 175                 for (i = 0; i < height; i++) {
 176                         for (j = 0; j < width; j++) {
 177                                 grey = *i1++;
 178                                 grey += *i1++;
 179                                 grey += *i1++;
 180 //                              CV_IMAGE_ELEM(image1, float, i, j) = grey / 76;
 181                                 CV_IMAGE_ELEM(image1, ushort, i, j) = grey / 3;
 182                                 grey = *i2++;
 183                                 grey += *i2++;
 184                                 grey += *i2++;
 185 //                              CV_IMAGE_ELEM(image2, float, i, j) = grey / 76;
 186                                 CV_IMAGE_ELEM(image2, ushort, i, j) = grey / 3;
 187                         }
 188                 }
 189 #if 0
 190 cvNamedWindow( "Image1", 0 );// Create a window for display.
 191 cvShowImage( "Image1", image1);                   // Show our image inside it.
 192 cvNamedWindow( "Image2", 0 );// Create a window for display.
 193 cvShowImage( "Image2", image2);                   // Show our image inside it.
 194 #endif
 195
 196
 197                 cvCalcOpticalFlowFarneback(image1, image2, flow1, pyrScale, levels, winsize, iterations, polyN, polySigma, flags);
 198                 cvCalcOpticalFlowFarneback(image2, image1, flow2, pyrScale, levels, winsize, iterations, polyN, polySigma, flags);
 199
 200 #ifndef TEST_INPUT
 201                 cvReleaseImage(&image1);
 202                 cvReleaseImage(&image2);
 203 #endif
 204         }
 205
 206         /* apply image to array using flow map */
 207         for (i = 0; i < height; i++) {
 208                 f = (const float *)(flow1->data.ptr + flow1->step * i);
 209                 for (j = 0; j < width; j++) {
 210                         x = j + round(*f++ * offset);
 211                         y = i + round(*f++ * offset);
 212                         if (j < border || i < border || j >= width-border || i >= height-border)
 213                                 continue;
 214                         if (x < 0 || x >= width || y < 0 || y >= height)
 215                                 continue;
 216                         array1[y*width+x] += (double)img1[(i*width+j)*3+0] / 65535;
 217                         array1[y*width+x + width*height] += (double)img1[(i*width+j)*3+1] / 65535;
 218                         array1[y*width+x + width*height*2] += (double)img1[(i*width+j)*3+2] / 65535;
 219                         array1[y*width+x + width*height*3] += 1.0;
 220                 }
 221         }
 222         for (i = 0; i < height; i++) {
 223                 f = (const float *)(flow2->data.ptr + flow2->step * i);
 224                 for (j = 0; j < width; j++) {
 225                         x = j + round(*f++ * (1-offset));
 226                         y = i + round(*f++ * (1-offset));
 227                         if (j < border || i < border || j >= width-border || i >= height-border)
 228                                 continue;
 229                         if (x < 0 || x >= width || y < 0 || y >= height)
 230                                 continue;
 231                         array2[y*width+x] += (double)img2[(i*width+j)*3+0] / 65535;
 232                         array2[y*width+x + width*height] += (double)img2[(i*width+j)*3+1] / 65535;
 233                         array2[y*width+x + width*height*2] += (double)img2[(i*width+j)*3+2] / 65535;
 234                         array2[y*width+x + width*height*3] += 1.0;
 235                 }
 236         }
 237
 238         /* mask unset pixles */
 239         for (i = 0; i < height; i++) {
 240                 for (j = 0; j < width; j++) {
 241                         alpha = array1[i*width+j + width*height*3];
 242                         if (alpha == 0.0) {
 243                                 array1[i*width+j] = 0.0;
 244                                 array1[i*width+j + width*height] = 0.0;
 245                                 array1[i*width+j + width*height*2] = 0.0;
 246                                 array1[i*width+j + width*height*3] = 0.0;
 247                                 /* mask */
 248                                 array1[i*width+j + width*height*4] = 1.0;
 249                         }
 250                         alpha = array2[i*width+j + width*height*3];
 251                         if (alpha == 0.0) {
 252                                 array2[i*width+j] = 0.0;
 253                                 array2[i*width+j + width*height] = 0.0;
 254                                 array2[i*width+j + width*height*2] = 0.0;
 255                                 array2[i*width+j + width*height*3] = 0.0;
 256                                 /* mask */
 257                                 array2[i*width+j + width*height*4] = 1.0;
 258                         }
 259                 }
 260         }
 261
 262 #ifndef TEST_SINGLE
 263         /* smooth image by interpolating missing (masked) pixles */
 264         smooth(array1, array1+width*height*4, width, height);
 265         smooth(array1+width*height, array1+width*height*4, width, height);
 266         smooth(array1+width*height*2, array1+width*height*4, width, height);
 267         smooth(array1+width*height*3, array1+width*height*4, width, height);
 268         smooth(array2, array2+width*height*4, width, height);
 269         smooth(array2+width*height, array2+width*height*4, width, height);
 270         smooth(array2+width*height*2, array2+width*height*4, width, height);
 271         smooth(array2+width*height*3, array2+width*height*4, width, height);
 272 #endif
 273
 274         /* normalize smoothed image and compose */
 275         for (i = 0; i < height; i++) {
 276                 for (j = 0; j < width; j++) {
 277                         alpha = array1[i*width+j + width*height*3];
 278                         if (alpha > 0.0) {
 279                                 array1[i*width+j] /= alpha;
 280                                 array1[i*width+j + width*height] /= alpha;
 281                                 array1[i*width+j + width*height*2] /= alpha;
 282                                 array1[i*width+j + width*height*3] = 1.0;
 283                         }
 284                         alpha = array2[i*width+j + width*height*3];
 285                         if (alpha > 0.0) {
 286                                 array2[i*width+j] /= alpha;
 287                                 array2[i*width+j + width*height] /= alpha;
 288                                 array2[i*width+j + width*height*2] /= alpha;
 289                                 array2[i*width+j + width*height*3] = 1.0;
 290                         }
 291 #ifndef TEST_SINGLE
 292                         array1[i*width+j] = (1.0-offset)*array1[i*width+j] + offset*array2[i*width+j];
 293                         array1[i*width+j + width*height] = (1.0-offset)*array1[i*width+j + width*height] + offset*array2[i*width+j + width*height];
 294                         array1[i*width+j + width*height*2] = (1.0-offset)*array1[i*width+j + width*height*2] + offset*array2[i*width+j + width*height*2];
 295 #endif
 296 #ifdef TEST_INPUT
 297                         if (i < (height>>1)) {
 298                                 array1[i*width+j] = (double)CV_IMAGE_ELEM(image2, ushort, i, j) / 65535;
 299                                 array1[i*width+j + width*height] = (double)CV_IMAGE_ELEM(image1, ushort, i, j) / 65535;
 300                                 array1[i*width+j + width*height*2] = 0;
 301                         }
 302 #endif
 303                 }
 304         }
 305
 306         array2img_short(array1, width, height, res, width, height, 0);
 307
 308         free(array1);
 309         free(array2);
 310
 311         return 0;
 312 }
 313
 314 static void print_help(const char *app)
 315 {
 316         printf("This tool will change frame rate using optical flow and interpolation.\n");
 317         printf("Usage: %s [options] <input> <first> <output> <first> <oldfps> <newfps>\n", app);
 318         printf(" -h --help            This help\n");
 319         printf(" -d --depth <bits>    Change color depth from default %d to given bits\n", save_depth);
 320         printf(" -b --border <pixles> Remove border pixles, if darker than image (try 1)\n");
 321         printf(" -w --window <size>   Change optical flow window size (default %d)\n", flow_window);
 322         printf(" -f --frames <number> Limit number of input frames (default infinite)\n");
 323         printf(" -i --interlace       Handle input image as interlaced image\n");
 324         printf("                      Note: oldfps refers to frames, not the individual fields\n");
 325         printf(" -e --even            Use even lines for first field, instead of odd lines\n");
 326         printf(" input                Input image sequence (e.g. input_%%05d.ppm)\n");
 327         printf(" output               Output image sequence (e.g. output_%%05d.ppm)\n");
 328         printf(" first                Index of first input and output image\n");
 329         printf(" oldfps               Old frame rate (complete frames in case of interlace)\n");
 330         printf(" newfps               New frame rate\n");
 331 }
 332
 333 static int handle_options(int argc, char **argv)
 334 {
 335         int skip_args = 0;
 336
 337         while (1) {
 338                 int option_index = 0, c;
 339                 static struct option long_options[] = {
 340                         {"help", 0, 0, 'h'},
 341                         {"depth", 1, 0, 'd'},
 342                         {"border", 1, 0, 'b'},
 343                         {"window", 1, 0, 'w'},
 344                         {"frames", 1, 0, 'f'},
 345                         {"interlace", 0, 0, 'i'},
 346                         {"even", 0, 0, 'e'},
 347                         {0, 0, 0, 0},
 348                 };
 349
 350                 c = getopt_long(argc, argv, "hd:b:w:f:ie", long_options, &option_index);
 351
 352                 if (c == -1)
 353                         break;
 354
 355                 switch (c) {
 356                 case 'h':
 357                         print_help(argv[0]);
 358                         exit(0);
 359                 case 'd':
 360                         save_depth = atoi(optarg);
 361                         skip_args += 2;
 362                         break;
 363                 case 'b':
 364                         border = atoi(optarg);
 365                         skip_args += 2;
 366                         break;
 367                 case 'w':
 368                         flow_window = atoi(optarg);
 369                         skip_args += 2;
 370                         break;
 371                 case 'f':
 372                         limit_frames = atoi(optarg);
 373                         skip_args += 2;
 374                         break;
 375                 case 'i':
 376                         interlace = 1;
 377                         skip_args++;
 378                         break;
 379                 case 'e':
 380                         swap = 1;
 381                         skip_args++;
 382                         break;
 383                 default:
 384                         break;
 385                 }
 386         }
 387
 388         return skip_args;
 389 }
 390
 391 int main(int argc, char *argv[])
 392 {
 393         unsigned short *img1 = NULL, *img2 = NULL, *res = NULL;
 394         int width, height, width2, height2;
 395         const char *inputname, *outputname;
 396         double oldfps, newfps, step, inputframe, baseframe, offset;
 397         int outputframe;
 398         int skip_args;
 399         const char *app_name = argv[0];
 400         int rc;
 401
 402         skip_args = handle_options(argc, argv);
 403         argc -= skip_args + 1;
 404         argv += skip_args + 1;
 405
 406         if (argc != 6) {
 407                 print_help(app_name);
 408                 return 0;
 409         }
 410
 411         inputname = argv[0];
 412         inputframe = atoi(argv[1]);
 413         outputname = argv[2];
 414         outputframe = atoi(argv[3]);
 415         oldfps = strtod(argv[4], NULL);
 416         newfps = strtod(argv[5], NULL);
 417
 418         step = oldfps / newfps;
 419         printf("Time Scale Factor: %.4f\n", step);
 420         printf("Optical Flow window: %d\n", flow_window);
 421         if (limit_frames)
 422                 printf("Limit number of input frames: %.0f\n", limit_frames);
 423
 424 next_frame:
 425         baseframe = floor(inputframe);
 426         offset = inputframe - baseframe;
 427         if (offset < 0.001)
 428                 offset = 0.0;
 429         else if (interlace && offset < 0.501 && offset > 0.499)
 430                 offset = 0.5;
 431         else if (offset > 0.999) {
 432                 offset = 0.0;
 433                 baseframe++;
 434         }
 435         if (offset == 0.0 || (interlace && offset == 0.5)) {
 436                 if (interlace) {
 437                         if (offset < 0.5)
 438                                 printf("Copy input frame %d (first field), output frame %d\n", (int)baseframe, outputframe);
 439                         else
 440                                 printf("Copy input frame %d (second field), output frame %d\n", (int)baseframe, outputframe);
 441                 } else
 442                         printf("Copy input frame %d, output frame %d\n", (int)baseframe, outputframe);
 443                 res = load_img(&width, &height, inputname, baseframe);
 444                 if (!res)
 445                         goto end;
 446                 if (interlace) {
 447                         if (offset < 0.5)
 448                                 field_img(width, height, res, (!swap)); // first field
 449                         else
 450                                 field_img(width, height, res, swap); // second field
 451                 }
 452                 rc = save_img(res, width, height, 0, outputname, outputframe);
 453                 if (rc < 0)
 454                         goto out;
 455         } else {
 456                 printf("Processing input frame %.4f, output frame %d\n", inputframe, outputframe);
 457                 img1 = load_img(&width, &height, inputname, baseframe);
 458                 if (!img1)
 459                         goto end;
 460                 if (interlace && offset < 0.5) {
 461                         field_img(width, height, img1, (!swap)); // first field of baseframe
 462                         img2 = load_img(&width2, &height2, inputname, baseframe);
 463                         if (!img2)
 464                                 goto end;
 465                         field_img(width2, height2, img2, swap); // second field of baseframe
 466                 } else {
 467                         img2 = load_img(&width2, &height2, inputname, baseframe+1);
 468                         if (!img2)
 469                                 goto end;
 470                         if (interlace) {
 471                                 field_img(width, height, img1, swap); // second field of baseframe
 472                                 field_img(width2, height2, img2, (!swap)); // first field of baseframe+1
 473                         }
 474                 }
 475                 if (width != width2) {
 476                         printf("Frame %d and %d have different width (%d != %d)\n", (int)baseframe, (int)baseframe+1, width, width2);
 477                         goto out;
 478                 }
 479                 if (height != height2) {
 480                         printf("Frame %d and %d have different height (%d != %d)\n", (int)baseframe, (int)baseframe+1, height, height2);
 481                         goto out;
 482                 }
 483                 if (interlace && (height & 1)) {
 484                         printf("Interlaced frame %d must have even number of lines (not %d)\n", (int)baseframe, height);
 485                         goto out;
 486                 }
 487                 res = malloc(width * height * 3 * sizeof(unsigned short));
 488                 if (!res)
 489                         goto out;
 490                 if (interlace) {
 491                         if (offset < 0.5)
 492                                 rc = interpolate(img1, img2, res, width, height, offset * 2.0, ((int)baseframe) * 2);
 493                         else
 494                                 rc = interpolate(img1, img2, res, width, height, (offset * 2.0) - 1.0, ((int)baseframe) * 2 + 1);
 495                 } else
 496                         rc = interpolate(img1, img2, res, width, height, offset, (int)baseframe);
 497                 if (rc < 0)
 498                         goto out;
 499                 rc = save_img(res, width, height, 0, outputname, outputframe);
 500                 if (rc < 0)
 501                         goto out;
 502         }
 503
 504         free(img1);
 505         img1 = NULL;
 506         free(img2);
 507         img2 = NULL;
 508         free(res);
 509         res = NULL;
 510
 511         outputframe++;
 512         inputframe += step;
 513         if (limit_frames) {
 514                 limit_frames -= step;
 515                 if (limit_frames <= 0)
 516                         goto out;
 517         }
 518         goto next_frame;
 519
 520 end:
 521         printf("We end here, since there is no more image to process\n");
 522
 523 out:
 524         free(img1);
 525         free(img2);
 526         free(res);
 527         return 0;
 528 }