Various fixes

- Common/R66: improve multithreading usage
- Compression: make Zlib codec usable for FTP (mode Z) (only using compatible clients, such as WaarpFtp4J based o FTP4J)
- Packaging: following issue #2 from fb clone on packaging using release profile
- Update dependencies (including issue #1 from fb clone)
- Try to fix and improve FTP concurrency (including issue #99)
- Fix documentation

Author: fredericBregier

README.md

@@ -62,6 +62,24 @@ cd Waarp-All
 mvn -P jre11 package
 ```
 
+If you want to build unofficial RPM/DEV/TGZ/ZIP and documentation, you can do as the following,
+ensuring you have already cloned and install using `pip` the repo for Sphinx template for Waarp
+`code.waarp.fr:2222/waarp/sphinx-template.git` with the following packages for Sphinx:
+
+- sphinx
+- sphinx-autobuild
+- sphinxcontrib-httpdomain
+  - Possibly fix the current version 1.6 to 1.7
+    - `sphinxcontrib/httpdomain.py`
+      - line 766
+      - `+ app.add_domain(HTTPDomain)`
+- sphinxcontrib-openapi
+- sphinx.ext.todo
+
+```sh
+mvn -P jre11,release package
+```
+
 You can use a JDK 11 (or higher) with `jre11` profile, and a JDK 8 with `jre8` or `jre6` profiles.
 
 `mvn -P jre11 package` also runs the full test suite, which takes quite some time (for more

WaarpCommon/src/main/java/org/waarp/common/database/DbSession.java

@@ -122,7 +122,8 @@ private void initialize(final DbModel dbModel, final String server,
     } catch (final SQLException ex) {
       setDisActive(true);
       // handle any errors
-      logger.error(CANNOT_CREATE_CONNECTION);
+      logger.error(CANNOT_CREATE_CONNECTION + " while already having {}",
+                   DbAdmin.getNbConnection());
       error(ex);
       if (getConn() != null) {
         try {
@@ -223,7 +224,8 @@ public final void setAutoCommit(final boolean autoCommit)
         getConn().setAutoCommit(autoCommit);
       } catch (final SQLException e) {
         // handle any errors
-        logger.error(CANNOT_CREATE_CONNECTION);
+        logger.error(CANNOT_CREATE_CONNECTION + " while already having {}",
+                     DbAdmin.getNbConnection());
         error(e);
         if (getConn() != null) {
           try {

WaarpCommon/src/main/java/org/waarp/common/file/DataBlock.java

@@ -21,6 +21,7 @@
 
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
+import org.waarp.common.utility.WaarpNettyUtil;
 
 /**
  * Main object implementing Data Block whaveter the mode, type, structure used.
@@ -139,11 +140,11 @@ public final void setBlock(final ByteBuf block) {
     this.block = new byte[byteCount];
     offsetBuf = 0;
     if (blockBuf != null) {
-      blockBuf.release();
+      WaarpNettyUtil.release(blockBuf);
       blockBuf = null;
     }
     block.readBytes(this.block);
-    block.release();
+    WaarpNettyUtil.release(block);
   }
 
   /**
@@ -165,8 +166,14 @@ public final void setBlock(final byte[] block, final int size) {
     if (isRESTART) {
       this.block = null;
       markers = new int[6];
-      for (int i = 0; i < 6; i++) {
-        markers[i] = block[i];
+      if (block == null) {
+        for (int i = 0; i < 6; i++) {
+          markers[i] = 0;
+        }
+      } else {
+        for (int i = 0; i < 6; i++) {
+          markers[i] = block[i];
+        }
       }
       byteCount = 6;
       return;
@@ -178,7 +185,7 @@ public final void setBlock(final byte[] block, final int size) {
       byteCount = size;
     }
     if (blockBuf != null) {
-      blockBuf.release();
+      WaarpNettyUtil.release(blockBuf);
       blockBuf = null;
     }
     offsetBuf = 0;
@@ -337,7 +344,7 @@ public final void setMarkers(final int[] markers) {
    */
   public final void clear() {
     if (blockBuf != null) {
-      blockBuf.release();
+      WaarpNettyUtil.release(blockBuf);
       blockBuf = null;
     }
     block = null;

WaarpCommon/src/main/java/org/waarp/common/file/filesystembased/FilesystemBasedFileImpl.java

@@ -276,7 +276,7 @@ public final String getFile() throws CommandAbstractException {
   }
 
   @Override
-  public boolean closeFile() throws CommandAbstractException {
+  public synchronized boolean closeFile() throws CommandAbstractException {
     if (fileInputStream != null) {
       FileUtils.close(fileInputStream);
       fileInputStream = null;
@@ -295,7 +295,8 @@ public boolean closeFile() throws CommandAbstractException {
   }
 
   @Override
-  public final boolean abortFile() throws CommandAbstractException {
+  public final synchronized boolean abortFile()
+      throws CommandAbstractException {
     if (isInWriting() &&
         ((FilesystemBasedFileParameterImpl) getSession().getFileParameter()).deleteOnAbort) {
       delete();
@@ -320,15 +321,15 @@ public long length() throws CommandAbstractException {
   }
 
   @Override
-  public final boolean isInReading() {
+  public final synchronized boolean isInReading() {
     if (!isReady) {
       return false;
     }
     return fileInputStream != null;
   }
 
   @Override
-  public final boolean isInWriting() {
+  public final synchronized boolean isInWriting() {
     if (!isReady) {
       return false;
     }
@@ -473,7 +474,7 @@ public boolean renameTo(final String path) throws CommandAbstractException {
   }
 
   @Override
-  public final DataBlock readDataBlock()
+  public final synchronized DataBlock readDataBlock()
       throws FileTransferException, FileEndOfTransferException {
     if (isReady) {
       return getByteBlock(getSession().getBlockSize());
@@ -482,7 +483,7 @@ public final DataBlock readDataBlock()
   }
 
   @Override
-  public final DataBlock readDataBlock(final byte[] bufferGiven)
+  public final synchronized DataBlock readDataBlock(final byte[] bufferGiven)
       throws FileTransferException, FileEndOfTransferException {
     if (isReady) {
       return getByteBlock(bufferGiven);
@@ -491,7 +492,7 @@ public final DataBlock readDataBlock(final byte[] bufferGiven)
   }
 
   @Override
-  public final void writeDataBlock(final DataBlock dataBlock)
+  public final synchronized void writeDataBlock(final DataBlock dataBlock)
       throws FileTransferException {
     if (isReady) {
       if (dataBlock.isEOF()) {
@@ -514,7 +515,7 @@ public final void writeDataBlock(final DataBlock dataBlock)
    *
    * @return the position
    */
-  public final long getPosition() {
+  public final synchronized long getPosition() {
     return position;
   }
 
@@ -526,7 +527,8 @@ public final long getPosition() {
    * @throws IOException
    */
   @Override
-  public final void setPosition(final long position) throws IOException {
+  public final synchronized void setPosition(final long position)
+      throws IOException {
     if (this.position != position) {
       this.position = position;
       if (fileInputStream != null) {
@@ -556,8 +558,9 @@ public final void setPosition(final long position) throws IOException {
    *
    * @throws FileTransferException
    */
-  private void writeBlock(final byte[] buffer, final int offset,
-                          final int length) throws FileTransferException {
+  private synchronized void writeBlock(final byte[] buffer, final int offset,
+                                       final int length)
+      throws FileTransferException {
     if (length > 0 && !isReady) {
       throw new FileTransferException(NO_FILE_IS_READY);
     }
@@ -597,8 +600,9 @@ private void writeBlock(final byte[] buffer, final int offset,
    *
    * @throws FileTransferException
    */
-  private void writeBlockEnd(final byte[] buffer, final int offset,
-                             final int length) throws FileTransferException {
+  private synchronized void writeBlockEnd(final byte[] buffer, final int offset,
+                                          final int length)
+      throws FileTransferException {
     writeBlock(buffer, offset, length);
     try {
       closeFile();
@@ -629,7 +633,7 @@ private void checkByteBufSize(final int size) {
    * @throws FileTransferException
    * @throws FileEndOfTransferException
    */
-  private DataBlock getByteBlock(final int sizeblock)
+  private synchronized DataBlock getByteBlock(final int sizeblock)
       throws FileTransferException, FileEndOfTransferException {
     if (!isReady) {
       throw new FileTransferException(NO_FILE_IS_READY);
@@ -656,7 +660,7 @@ private DataBlock getByteBlock(final int sizeblock)
    * @throws FileTransferException
    * @throws FileEndOfTransferException
    */
-  private DataBlock getByteBlock(final byte[] bufferGiven)
+  private synchronized DataBlock getByteBlock(final byte[] bufferGiven)
       throws FileTransferException, FileEndOfTransferException {
     if (!isReady) {
       throw new FileTransferException(NO_FILE_IS_READY);

WaarpCommon/src/main/java/org/waarp/common/guid/GUID.java

@@ -196,7 +196,7 @@ public GUID(final int tenantId, final int platformId) {
       count = ++counter;
       if (count > 0xFFFFFF) {
         try {
-          FORSYNC.wait(1);//NOSONAR
+          Thread.sleep(1);//NOSONAR
         } catch (final InterruptedException e) {//NOSONAR
           // ignore
           SysErrLogger.FAKE_LOGGER.ignoreLog(e);

WaarpCommon/src/main/java/org/waarp/common/utility/WaarpNettyUtil.java

@@ -29,9 +29,14 @@
 import io.netty.channel.WriteBufferWaterMark;
 import io.netty.channel.socket.nio.NioServerSocketChannel;
 import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.util.IllegalReferenceCountException;
 import io.netty.util.concurrent.Future;
 import org.waarp.common.logging.SysErrLogger;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+
 /**
  * Utility class for Netty usage
  */
@@ -159,6 +164,58 @@ public static void setServerBootstrap(final ServerBootstrap bootstrap,
     }
   }
 
+  /**
+   * Checks to see if a specific port is available.
+   *
+   * @param port the port to check for availability
+   */
+  public static boolean availablePort(final int port) {
+    ServerSocket ss = null;
+    try {
+      ss = new ServerSocket(port); // NOSONAR
+      ss.setReuseAddress(true);
+      return true;
+    } catch (final IOException e) {
+      SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+    } finally {
+      if (ss != null) {
+        try {
+          ss.close();
+        } catch (final Exception e) {
+          SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+        }
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Checks to see if a specific port is available.
+   *
+   * @param port the port to check for availability
+   * @param localAddress the associated localAddress to use
+   */
+  public static boolean availablePort(final int port,
+                                      final InetAddress localAddress) {
+    ServerSocket ss = null;
+    try {
+      ss = new ServerSocket(port, 0, localAddress); // NOSONAR
+      ss.setReuseAddress(true);
+      return true;
+    } catch (final IOException e) {
+      SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+    } finally {
+      if (ss != null) {
+        try {
+          ss.close();
+        } catch (final Exception e) {
+          SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+        }
+      }
+    }
+    return false;
+  }
+
   /**
    * @param future
    *
@@ -225,7 +282,12 @@ public static boolean release(final ByteBuf byteBuf) {
     if (byteBuf == null || byteBuf.refCnt() <= 0) {
       return true;
     }
-    return byteBuf.release();
+    try {
+      return byteBuf.release();
+    } catch (final IllegalReferenceCountException e) {
+      SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+      return true;
+    }
   }
 
   /**
@@ -236,7 +298,26 @@ public static boolean release(final ByteBuf byteBuf) {
   public static void releaseCompletely(final ByteBuf byteBuf) {
     if (byteBuf != null && byteBuf.refCnt() != 0) {
       final int refCnt = byteBuf.refCnt();
-      byteBuf.release(refCnt);
+      try {
+        byteBuf.release(refCnt);
+      } catch (final IllegalReferenceCountException e) {
+        SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+      }
+    }
+  }
+
+  /**
+   * Retain this ByteBuf
+   *
+   * @param byteBuf
+   */
+  public static void retain(final ByteBuf byteBuf) {
+    if (byteBuf != null) {
+      try {
+        byteBuf.retain();
+      } catch (final IllegalReferenceCountException e) {
+        SysErrLogger.FAKE_LOGGER.ignoreLog(e);
+      }
     }
   }