Fixed bugs after dnn refactoring. Added new tests for layers (#1142)

* Fixed bugs after dnn refactoring. Added new tests for layers

* Fixed torch tests

Author: arrybn

modules/dnn/src/dnn.cpp

@@ -607,7 +607,7 @@ void Net::setBlob(String outputName, const Mat &blob_)
     MatSize prevShape = ld.outputBlobs[pin.oid].size;
     ld.outputBlobs[pin.oid] = blob_.clone();
 
-    impl->netWasAllocated = prevShape == blob_.size;
+    impl->netWasAllocated = impl->netWasAllocated && prevShape == blob_.size;
 }
 
 Mat Net::getBlob(String outputName)

modules/dnn/src/layers/op_im2col.cpp

@@ -153,12 +153,7 @@ class im2row_CpuPBody : public cv::ParallelLoopBody
         t.width_col = width_col;
         t.channels_col = channels * kernel_h * kernel_w;
 
-        int total = t.height_col*t.width_col;
-#if 1
-        t(Range(0, total));
-#else
-        cv::parallel_for_(Range(0, total), t, 16);
-#endif
+        cv::parallel_for_(Range(0, t.height_col*t.width_col), t, 16);
     }
 
     virtual void operator ()(const Range &r) const
@@ -203,7 +198,6 @@ class im2row_CpuPBody : public cv::ParallelLoopBody
             }
             else
             {
-                memset(data_col_, 0, kw*kh*channels*sizeof(data_col_[0]));
                 for(int i_c = 0; i_c < channels; i_c++)
                 {
                     int channels_offset = i_c * width * height;
@@ -242,7 +236,6 @@ void im2row(const float* data_im, int channels, int height, int width,
 }
 
 
-#if 0
 template <typename Dtype>
 class col2im_CpuPBody : public cv::ParallelLoopBody
 {
@@ -312,7 +305,6 @@ class col2im_CpuPBody : public cv::ParallelLoopBody
         }
     }
 };
-#endif
 
 //single-threaded version
 template <typename Dtype>
@@ -360,9 +352,15 @@ void col2im(const float* data_col, int channels, int height, int width,
             int stride_h, int stride_w, int dilation_h, int dilation_w,
             float* data_im, const int* ofsbuf)
 {
-    //col2im_CpuPBody<float>::run(data_col, channels, height, width, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, data_im);
+    (void)dilation_h;
+    (void)dilation_w;
+    (void)ofsbuf;
+    col2im_CpuPBody<float>::run(data_col, channels, height, width, kernel_h,
+                                kernel_w, pad_h, pad_w, stride_h, stride_w, data_im);
+#if 0
     col2im_cpu(data_col, channels, height, width, kernel_h, kernel_w, pad_h, pad_w,
                stride_h, stride_w, dilation_h, dilation_w, data_im, ofsbuf);
+#endif
 }
 
 }

modules/dnn/src/torch/THDiskFile.cpp

@@ -13,6 +13,7 @@ typedef struct THDiskFile__
     FILE *handle;
     char *name;
     int isNativeEncoding;
+    int longSize;
 
 } THDiskFile;
 
@@ -172,9 +173,17 @@ static void THDiskFile_seek(THFile *self, long position)
   THDiskFile *dfself = (THDiskFile*)(self);
 
   THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
-  THArgCheck(position >= 0, 2, "position must be positive");
 
-  if(fseek(dfself->handle, position, SEEK_SET) < 0)
+#if defined(_WIN64)
+  THArgCheck(position <= (_int64)INT64_MAX, 2, "position must be smaller than INT64_MAX");
+  if(_fseeki64(dfself->handle, (__int64)position, SEEK_SET) < 0)
+#elif defined(_WIN32)
+  THArgCheck(position <= (long)LONG_MAX, 2, "position must be smaller than LONG_MAX");
+  if(fseek(dfself->handle, (long)position, SEEK_SET) < 0)
+#else
+  THArgCheck(position <= (long)LLONG_MAX, 2, "position must be smaller than LLONG_MAX");
+  if(fseeko(dfself->handle, (off_t)position, SEEK_SET) < 0)
+#endif
   {
     dfself->file.hasError = 1;
     if(!dfself->file.isQuiet)
@@ -188,7 +197,13 @@ static void THDiskFile_seekEnd(THFile *self)
 
   THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
 
+#if defined(_WIN64)
+  if(_fseeki64(dfself->handle, 0L, SEEK_END) < 0)
+#elif defined(_WIN32)
   if(fseek(dfself->handle, 0L, SEEK_END) < 0)
+#else
+  if(fseeko(dfself->handle, 0L, SEEK_END) < 0)
+#endif
   {
     dfself->file.hasError = 1;
     if(!dfself->file.isQuiet)
@@ -200,7 +215,20 @@ static long THDiskFile_position(THFile *self)
 {
   THDiskFile *dfself = (THDiskFile*)(self);
   THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
-  return ftell(dfself->handle);
+
+#if defined(_WIN64)
+  __int64 offset = _ftelli64(dfself->handle);
+#elif defined(_WIN32)
+  long offset = ftell(dfself->handle);
+#else
+  off_t offset = ftello(dfself->handle);
+#endif
+  if (offset > -1)
+      return (long)offset;
+  else if(!dfself->file.isQuiet)
+      THError("unable to obtain disk file offset (maybe a long overflow occurred)");
+
+  return 0;
 }
 
 static void THDiskFile_close(THFile *self)
@@ -274,6 +302,23 @@ void THDiskFile_bigEndianEncoding(THFile *self)
 
 /* End of Little and Big Endian Stuff */
 
+void THDiskFile_longSize(THFile *self, int size)
+{
+  THDiskFile *dfself = (THDiskFile*)(self);
+  THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+  THArgCheck(size == 0 || size == 4 || size == 8, 1, "Invalid long size specified");
+  dfself->longSize = size;
+}
+
+void THDiskFile_noBuffer(THFile *self)
+{
+  THDiskFile *dfself = (THDiskFile*)(self);
+  THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+  if (setvbuf(dfself->handle, NULL, _IONBF, 0)) {
+    THError("error: cannot disable buffer");
+  }
+}
+
 static void THDiskFile_free(THFile *self)
 {
   THDiskFile *dfself = (THDiskFile*)(self);
@@ -302,12 +347,12 @@ READ_WRITE_METHODS(short, Short,
                    int ret = fprintf(dfself->handle, "%hd", data[i]); if(ret <= 0) break; else nwrite++)
 
 READ_WRITE_METHODS(int, Int,
-                   int ret = fscanf(dfself->handle, "%d", &data[i]); if(ret <= 0) break; else nread++,
+                   int ret = fscanf(dfself->handle, "%d\n\r", &data[i]); if(ret <= 0) break; else nread++,
                    int ret = fprintf(dfself->handle, "%d", data[i]); if(ret <= 0) break; else nwrite++)
 
-READ_WRITE_METHODS(long, Long,
+/*READ_WRITE_METHODS(long, Long,
                    int ret = fscanf(dfself->handle, "%ld", &data[i]); if(ret <= 0) break; else nread++,
-                   int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++)
+                   int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++)*/
 
 READ_WRITE_METHODS(float, Float,
                    int ret = fscanf(dfself->handle, "%g", &data[i]); if(ret <= 0) break; else nread++,
@@ -317,6 +362,146 @@ READ_WRITE_METHODS(double, Double,
                    int ret = fscanf(dfself->handle, "%lg", &data[i]); if(ret <= 0) break; else nread++,
                    int ret = fprintf(dfself->handle, "%.17g", data[i]); if(ret <= 0) break; else nwrite++)
 
+
+/* For Long we need to rewrite everything, because of the special management of longSize */
+static long THDiskFile_readLong(THFile *self, int64 *data, long n)
+{
+  THDiskFile *dfself = (THDiskFile*)(self);
+  long nread = 0L;
+
+  THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+  THArgCheck(dfself->file.isReadable, 1, "attempt to read in a write-only file");
+
+  if(dfself->file.isBinary)
+  {
+    if(dfself->longSize == 0 || dfself->longSize == sizeof(int64))
+    {
+      nread = fread__(data, sizeof(int64), n, dfself->handle);
+      if(!dfself->isNativeEncoding && (sizeof(int64) > 1) && (nread > 0))
+        THDiskFile_reverseMemory(data, data, sizeof(int64), nread);
+    } else if(dfself->longSize == 4)
+    {
+      nread = fread__(data, 4, n, dfself->handle);
+      if(!dfself->isNativeEncoding && (nread > 0))
+        THDiskFile_reverseMemory(data, data, 4, nread);
+      long i;
+      for(i = nread; i > 0; i--)
+        data[i-1] = ((int *)data)[i-1];
+    }
+    else /* if(dfself->longSize == 8) */
+    {
+      int big_endian = !THDiskFile_isLittleEndianCPU();
+      int32_t *buffer = (int32_t*)THAlloc(8*n);
+      nread = fread__(buffer, 8, n, dfself->handle);
+      long i;
+      for(i = nread; i > 0; i--)
+        data[i-1] = buffer[2*(i-1) + big_endian];
+      THFree(buffer);
+      if(!dfself->isNativeEncoding && (nread > 0))
+        THDiskFile_reverseMemory(data, data, 4, nread);
+     }
+  }
+  else
+  {
+    long i;
+    for(i = 0; i < n; i++)
+    {
+      long d;
+      int ret = fscanf(dfself->handle, "%ld", &d); if(ret <= 0) break; else nread++;
+      data[i] = d;
+    }
+    if(dfself->file.isAutoSpacing && (n > 0))
+    {
+      int c = fgetc(dfself->handle);
+      if( (c != '\n') && (c != EOF) )
+        ungetc(c, dfself->handle);
+    }
+  }
+
+  if(nread != n)
+  {
+    dfself->file.hasError = 1; /* shouldn't we put hasError to 0 all the time ? */
+    if(!dfself->file.isQuiet)
+      THError("read error: read %d blocks instead of %d", nread, n);
+  }
+
+  return nread;
+}
+
+static long THDiskFile_writeLong(THFile *self, int64 *data, long n)
+{
+  THDiskFile *dfself = (THDiskFile*)(self);
+  long nwrite = 0L;
+
+  THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+  THArgCheck(dfself->file.isWritable, 1, "attempt to write in a read-only file");
+
+  if(dfself->file.isBinary)
+  {
+    if(dfself->longSize == 0 || dfself->longSize == sizeof(long))
+    {
+      if(dfself->isNativeEncoding)
+      {
+        nwrite = fwrite(data, sizeof(long), n, dfself->handle);
+      }
+      else
+      {
+        char *buffer = (char*)THAlloc(sizeof(long)*n);
+        THDiskFile_reverseMemory(buffer, data, sizeof(long), n);
+        nwrite = fwrite(buffer, sizeof(long), n, dfself->handle);
+        THFree(buffer);
+      }
+    } else if(dfself->longSize == 4)
+    {
+      int32_t *buffer = (int32_t *)THAlloc(4*n);
+      long i;
+      for(i = 0; i < n; i++)
+        buffer[i] = data[i];
+      if(!dfself->isNativeEncoding)
+        THDiskFile_reverseMemory(buffer, buffer, 4, n);
+      nwrite = fwrite(buffer, 4, n, dfself->handle);
+      THFree(buffer);
+    }
+    else /* if(dfself->longSize == 8) */
+    {
+      int big_endian = !THDiskFile_isLittleEndianCPU();
+      int32_t *buffer = (int32_t*)THAlloc(8*n);
+      long i;
+      for(i = 0; i < n; i++)
+      {
+        buffer[2*i + !big_endian] = 0;
+        buffer[2*i + big_endian] = data[i];
+      }
+      if(!dfself->isNativeEncoding)
+        THDiskFile_reverseMemory(buffer, buffer, 8, n);
+      nwrite = fwrite(buffer, 8, n, dfself->handle);
+      THFree(buffer);
+    }
+  }
+  else
+  {
+    long i;
+    for(i = 0; i < n; i++)
+    {
+      long res = 0;
+      int ret = fprintf(dfself->handle, "%ld", res); data[i] = res; if(ret <= 0) break; else nwrite++;
+      if( dfself->file.isAutoSpacing && (i < n-1) )
+        fprintf(dfself->handle, " ");
+    }
+    if(dfself->file.isAutoSpacing && (n > 0))
+      fprintf(dfself->handle, "\n");
+  }
+
+  if(nwrite != n)
+  {
+    dfself->file.hasError = 1;
+    if(!dfself->file.isQuiet)
+      THError("write error: wrote %d blocks instead of %d", nwrite, n);
+  }
+
+  return nwrite;
+}
+
 static long THDiskFile_readString(THFile *self, const char *format, char **str_)
 {
   THDiskFile *dfself = (THDiskFile*)(self);
@@ -494,6 +679,7 @@ THFile *THDiskFile_new(const char *name, const char *mode, int isQuiet)
   self->name = (char*)THAlloc(strlen(name)+1);
   strcpy(self->name, name);
   self->isNativeEncoding = 1;
+  self->longSize = 0;
 
   self->file.vtable = &vtable;
   self->file.isQuiet = isQuiet;
@@ -595,6 +781,7 @@ THFile *THPipeFile_new(const char *name, const char *mode, int isQuiet)
   self->name = (char*)THAlloc(strlen(name)+1);
   strcpy(self->name, name);
   self->isNativeEncoding = 1;
+  self->longSize = 0;
 
   self->file.vtable = &vtable;
   self->file.isQuiet = isQuiet;

modules/dnn/src/torch/THDiskFile.h

@@ -13,5 +13,7 @@ TH_API int THDiskFile_isBigEndianCPU(void);
 TH_API void THDiskFile_nativeEndianEncoding(THFile *self);
 TH_API void THDiskFile_littleEndianEncoding(THFile *self);
 TH_API void THDiskFile_bigEndianEncoding(THFile *self);
+TH_API void THDiskFile_longSize(THFile *self, int size);
+TH_API void THDiskFile_noBuffer(THFile *self);
 
 #endif

modules/dnn/src/torch/THFile.cpp

@@ -1,7 +1,7 @@
 #if defined(ENABLE_TORCH_IMPORTER) && ENABLE_TORCH_IMPORTER
 #include "THFile.h"
 #include "THFilePrivate.h"
-
+#include <opencv2/core.hpp>
 extern "C"
 {
 
@@ -20,7 +20,7 @@ IMPLEMENT_THFILE_RW(Byte, unsigned char)
 IMPLEMENT_THFILE_RW(Char, char)
 IMPLEMENT_THFILE_RW(Short, short)
 IMPLEMENT_THFILE_RW(Int, int)
-IMPLEMENT_THFILE_RW(Long, long)
+IMPLEMENT_THFILE_RW(Long, int64)
 IMPLEMENT_THFILE_RW(Float, float)
 IMPLEMENT_THFILE_RW(Double, double)
 
@@ -134,7 +134,7 @@ IMPLEMENT_THFILE_SCALAR(Byte, unsigned char)
 IMPLEMENT_THFILE_SCALAR(Char, char)
 IMPLEMENT_THFILE_SCALAR(Short, short)
 IMPLEMENT_THFILE_SCALAR(Int, int)
-IMPLEMENT_THFILE_SCALAR(Long, long)
+IMPLEMENT_THFILE_SCALAR(Long, int64)
 IMPLEMENT_THFILE_SCALAR(Float, float)
 IMPLEMENT_THFILE_SCALAR(Double, double)