Coverity: Argument cannot be negative

1. Due to not checking the result of fseek(), it is possible to try to
   malloc() -1 bytes of storage. Checking the return from fseek() and
   erring if negative.
2. Changing the check between the result of fseek() and fread() to match
   signedness. Adding some casting, as at that point the fseek() result
   is always positive.

Fixes CIDs:
  573009 572928 572868

Author: ejohnstown

apps/wolfssh/common.c

@@ -76,7 +76,7 @@ static int load_der_file(const char* filename, byte** out, word32* outSz)
 {
     WFILE* file;
     byte* in;
-    word32 inSz;
+    long inSz;
     int ret;
 
     if (filename == NULL || out == NULL || outSz == NULL)
@@ -90,10 +90,10 @@ static int load_der_file(const char* filename, byte** out, word32* outSz)
         WFCLOSE(NULL, file);
         return -1;
     }
-    inSz = (word32)WFTELL(NULL, file);
+    inSz = WFTELL(NULL, file);
     WREWIND(NULL, file);
 
-    if (inSz == 0) {
+    if (inSz <= 0) {
         WFCLOSE(NULL, file);
         return -1;
     }
@@ -105,7 +105,7 @@ static int load_der_file(const char* filename, byte** out, word32* outSz)
     }
 
     ret = (int)WFREAD(NULL, in, 1, inSz, file);
-    if (ret <= 0 || (word32)ret != inSz) {
+    if (ret <= 0 || (word32)ret != (word32)inSz) {
         ret = -1;
         WFREE(in, NULL, 0);
         in = 0;

apps/wolfsshd/wolfsshd.c

@@ -242,7 +242,7 @@ static byte* getBufferFromFile(const char* fileName, word32* bufSz, void* heap)
 {
     FILE* file;
     byte* buf = NULL;
-    word32 fileSz;
+    long fileSz;
     word32 readSz;
 
     WOLFSSH_UNUSED(heap);
@@ -252,13 +252,17 @@ static byte* getBufferFromFile(const char* fileName, word32* bufSz, void* heap)
     if (WFOPEN(NULL, &file, fileName, "rb") != 0)
         return NULL;
     WFSEEK(NULL, file, 0, WSEEK_END);
-    fileSz = (word32)WFTELL(NULL, file);
+    fileSz = WFTELL(NULL, file);
+    if (fileSz < 0) {
+        WFCLOSE(NULL, file);
+        return NULL;
+    }
     WREWIND(NULL, file);
 
     buf = (byte*)WMALLOC(fileSz + 1, heap, DYNTYPE_SSHD);
     if (buf != NULL) {
         readSz = (word32)WFREAD(NULL, buf, 1, fileSz, file);
-        if (readSz < fileSz) {
+        if (readSz < (size_t)fileSz) {
             WFCLOSE(NULL, file);
             WFREE(buf, heap, DYNTYPE_SSHD);
             return NULL;

examples/client/common.c

@@ -262,7 +262,7 @@ static int load_der_file(const char* filename, byte** out, word32* outSz,
 {
     WFILE* file;
     byte* in;
-    word32 inSz;
+    long inSz;
     int ret;
 
     if (filename == NULL || out == NULL || outSz == NULL)
@@ -276,13 +276,12 @@ static int load_der_file(const char* filename, byte** out, word32* outSz,
         WFCLOSE(NULL, file);
         return -1;
     }
-    inSz = (word32)WFTELL(NULL, file);
-    WREWIND(NULL, file);
-
-    if (inSz == 0) {
+    inSz = WFTELL(NULL, file);
+    if (inSz <= 0) {
         WFCLOSE(NULL, file);
         return -1;
     }
+    WREWIND(NULL, file);
 
     in = (byte*)WMALLOC(inSz, heap, 0);
     if (in == NULL) {
@@ -291,7 +290,7 @@ static int load_der_file(const char* filename, byte** out, word32* outSz,
     }
 
     ret = (int)WFREAD(NULL, in, 1, inSz, file);
-    if (ret <= 0 || (word32)ret != inSz) {
+    if (ret <= 0 || (long)ret != inSz) {
         ret = -1;
         WFREE(in, heap, 0);
         in = 0;

tests/api.c

@@ -504,26 +504,38 @@ static int load_file(const char* filename, byte** buf, word32* bufSz)
     }
 
     if (ret == 0) {
-        fseek(f, 0, XSEEK_END);
-        *bufSz = (word32)ftell(f);
-        rewind(f);
+        ret = fseek(f, 0, XSEEK_END);
+        if (ret < 0)
+            ret = -3;
+    }
+
+    if (ret == 0) {
+        long sz = ftell(f);
+        if (sz < 0)
+            ret = -4;
+        else
+            *bufSz = (word32)sz;
     }
 
     if (ret == 0) {
+        rewind(f);
         *buf = (byte*)malloc(*bufSz);
         if (*buf == NULL)
-            ret = -3;
+            ret = -5;
     }
 
     if (ret == 0) {
-        int readSz;
-        readSz = (int)fread(*buf, 1, *bufSz, f);
-        if (readSz < (int)*bufSz)
-            ret = -4;
+        size_t readSz;
+        readSz = fread(*buf, 1, *bufSz, f);
+        if (readSz < *bufSz)
+            ret = -6;
     }
 
-    if (f != NULL)
-        fclose(f);
+    if (f != NULL) {
+        ret = fclose(f);
+        if (ret < 0)
+            ret = -7;
+    }
 
     return ret;
 }