Merge pull request #19 from grinya007/dev

Vim plugins => docker image, Dockerfile build stage removed, fixes

Author: grinya007

Dockerfile

@@ -1,54 +1,53 @@
-FROM postgres:14 AS base
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3 python3-pip postgresql-plpython3-14 less vim && \
-    apt-get remove --purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
-    rm -rf /usr/share/man /tmp/* /var/cache/apt/archives/* /var/lib/apt/lists/* /var/lib/dpkg/info/*
+FROM postgres:14
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        python3 \
+        python3-pandas \
+        python3-pip \
+        python3-psycopg2 \
+        postgresql-plpython3-14 \
+        curl \
+        git \
+        less \
+        vim \
+        vim-airline \
+    && apt-get remove --purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+    && rm -rf /usr/share/man /tmp/* /var/cache/apt/archives/* /var/lib/apt/lists/* /var/lib/dpkg/info/*
 
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
-
-FROM base AS build
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential python3-dev postgresql-server-dev-14 && \
-    apt-get remove --purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false && \
-    rm -rf /usr/share/man /tmp/* /var/cache/apt/archives/* /var/lib/apt/lists/* /var/lib/dpkg/info/*
-
-
-RUN mkdir /pip
-ENV PIP_TARGET=/pip
-ENV PYTHONPATH=/pip
-RUN pip install pandas psycopg2 pgcli
-
-
-FROM base
+RUN pip install pgcli
 
 ENV EDITOR=vim
 ENV PAGER="less -S"
 ENV PYTHONPYCACHEPREFIX=/tmp
 ENV POSTGRES_HOST_AUTH_METHOD=trust
 ENV TABLES_META=/tmp/tables_meta.pickle
 ENV CSV_DIR=/csv
-ENV PIP_TARGET=/pip
-ENV PYTHONPATH=/pip
-ENV PATH="$PATH:/pip/bin"
-
-RUN mkdir /pip
-RUN mkdir /csv
-RUN mkdir /src
-COPY src/* /src/
-WORKDIR /src
-
-COPY --from=build /pip/ /pip/
 
 USER postgres
 COPY db/init.sql /docker-entrypoint-initdb.d/
 RUN docker-entrypoint.sh
 
 USER root
-RUN mkdir -p /root/.config/pgcli
-COPY db/pgcli_config /root/.config/pgcli/config
+ENV HOME=/root
+RUN mkdir -p $HOME/.config/pgcli
+COPY db/pgcli_config $HOME/.config/pgcli/config
+
+# Vim plugins
+ENV GIT_SSL_NO_VERIFY=true
+RUN mkdir -p $HOME/.vim/pack/plugins/start
+RUN git clone --depth=1 https://github.com/nanotech/jellybeans.vim.git $HOME/.vim/pack/plugins/start/jellybeans
+RUN git clone --depth=1 https://github.com/ervandew/supertab.git $HOME/.vim/pack/plugins/start/supertab
+RUN git clone --depth=1 https://github.com/preservim/nerdcommenter.git $HOME/.vim/pack/plugins/start/nerdcommenter
+
+COPY .vimrc $HOME/
+
+RUN mkdir /csv
+RUN mkdir /src
+COPY src/* /src/
+WORKDIR /src
 
 RUN python load.py --just-compile
 RUN python dump.py --just-compile

src/dump.py

@@ -51,8 +51,14 @@ def dump(self, task: dict) -> None:
         with task['file'].open('w') as f:
             f.write(','.join(task['fields']) + "\n")
             conn.cursor().copy_expert(f"copy {task['copy_expr']} to stdout csv", f)
-            print(f"Done dumping [{task['filename']}]")
-
+        if task['remove_newline']:
+            with task['file'].open('rb+') as f:
+                f.seek(-1, os.SEEK_END)
+                if f.read(1) == b'\n':
+                    f.seek(-1, os.SEEK_END)
+                    f.truncate()
+
+        print(f"Done dumping [{task['filename']}]")
         conn.cursor().close()
         conn.close()
 

src/load.py

@@ -7,7 +7,6 @@
 import sys
 
 from time import time, sleep
-from pandas.core.frame import DataFrame
 from pandas.core.series import Series
 from pathlib import Path
 from psycopg2 import OperationalError
@@ -23,23 +22,35 @@
 CSV_DIR = os.getenv('CSV_DIR')
 TABLES_META = os.getenv('TABLES_META')
 
+
 def simplify_name(fname: str) -> str:
     fname = re.sub('\.csv$', '', fname)
     fname = re.sub('[^a-zA-Z0-9]', '_', fname)
     return fname.lower()
 
+
 def copy_expression(tname: str, fields: Series) -> str:
     tfields = list()
     for field, ftype in fields.items():
         tfield = simplify_name(field)
         if ftype == 'bool':
-            tfields.append(f'case when "{tfield}" = true then \'True\' else \'False\' end as "{tfield}"')
+            tfields.append(
+                f'case when "{tfield}" = true then \'TRUE\''
+                f' else \'FALSE\' end as "{tfield}"'
+            )
+        elif ftype == 'float64':
+            tfields.append(
+                f'case when "{tfield}"::text not like \'%.%\''
+                f' then "{tfield}"::text||\'.0\''
+                f' else "{tfield}"::text end as "{tfield}"'
+            )
         else:
             tfields.append(f'"{tfield}"')
 
     tfields = ','.join(tfields)
     return f"(select {tfields} from {tname} order by id)"
 
+
 def create_table(conn: connection, fname: str, fields: Series) -> dict:
     tname = simplify_name(fname)
     meta = {
@@ -48,6 +59,7 @@ def create_table(conn: connection, fname: str, fields: Series) -> dict:
         'filename': fname,
         'fields': [],
         'tablefields': [],
+        'remove_newline': True,
     }
 
     tfields = list()
@@ -61,9 +73,13 @@ def create_table(conn: connection, fname: str, fields: Series) -> dict:
 
         tfields.append(f'"{tfield}" {DATA_TYPES[ftype.name]}')
 
-    conn.cursor().execute(f"create table {tname} (id serial, {','.join(tfields)})")
+    conn.cursor().execute(
+        f"create table {tname}"
+        f" (id serial, {','.join(tfields)})"
+    )
     return meta
 
+
 def load(conn: connection, csv_file: Path) -> dict:
     df = pd.read_csv(csv_file, iterator=True, chunksize=10000)
 
@@ -76,7 +92,14 @@ def load(conn: connection, csv_file: Path) -> dict:
 
     fields = ','.join([f'"{f}"' for f in meta['tablefields']])
     with csv_file.open('r') as f:
-        conn.cursor().copy_expert(f"copy {meta['tablename']}({fields}) from stdin with header csv", f)
+        conn.cursor().copy_expert(
+            f"copy {meta['tablename']}({fields})"
+            f" from stdin with header csv", f
+        )
+    with csv_file.open('rb') as f:
+        f.seek(-1, os.SEEK_END)
+        if f.read(1) == b'\n':
+            meta['remove_newline'] = False
 
     return meta
 
@@ -86,7 +109,8 @@ def load(conn: connection, csv_file: Path) -> dict:
     parser.add_argument(
         "--just-compile",
         action="store_true",
-        help="Don't do actual load but just compile the script to speedup the first load",
+        help="Don't do actual load but just compile"
+             " the script to speedup the first load",
     )
     args = parser.parse_args()
     if args.just_compile: