First release

Signed-off-by: Zhou, Cheng <[email protected]>

Author: czhou26

CMakeLists.txt

@@ -0,0 +1,14 @@
+cmake_minimum_required (VERSION 2.8)
+
+Project(OVC NONE)
+
+file(GLOB dirs "deployment" "*")
+list(REMOVE_DUPLICATES dirs)
+foreach(dir ${dirs})
+    if(EXISTS ${dir}/CMakeLists.txt)
+        add_subdirectory(${dir})
+    endif()
+endforeach()
+
+# legal message
+execute_process(COMMAND printf "\nThis script will build third party components licensed under various open source licenses into your container images. The terms under which those components may be used and distributed can be found with the license document that is provided with those components. Please familiarize yourself with those terms to ensure your distribution of those components complies with the terms of those licenses.\n\n")

LICENSE

@@ -0,0 +1,14 @@
+This project is for Open Visual Cloud CDN Transcode E2E Sample.
+
+## Key ingredients in this repo
+- Dockerfiles
+- CMakefiles
+- Configuration files
+- Helper scripts
+- Reference Architecture document
+- User Guide (Getting Started Guide) document
+
+## How to get started?
+- Refer to "doc/CDN_Transcode_Sample_RA.md" to understand the design.
+- Refer to "doc/CDN_Transcode_Sample_Getting_Started_Guide.md" to setup and run the sample.
+

README.md

@@ -0,0 +1,4 @@
+CMakeLists.txt
+*.sh
+*.m4
+test/*

cdn-server/.dockerignore

@@ -0,0 +1,2 @@
+set(service "ovc_cdn_service")
+include("${CMAKE_SOURCE_DIR}/script/service.cmake")

cdn-server/CMakeLists.txt

@@ -0,0 +1,73 @@
+
+FROM ubuntu:18.04 AS build
+WORKDIR /home
+
+# COMMON BUILD TOOLS
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends build-essential autoconf make git wget pciutils cpio libtool lsb-release ca-certificates pkg-config bison flex
+
+# Install cmake
+ARG CMAKE_VER=3.13.1
+ARG CMAKE_REPO=https://cmake.org/files
+RUN wget -O - ${CMAKE_REPO}/v${CMAKE_VER%.*}/cmake-${CMAKE_VER}.tar.gz | tar xz && \
+    cd cmake-${CMAKE_VER} && \
+    ./bootstrap --prefix="/usr" && \
+    make -j8 && \
+    make install
+
+# Install automake, use version 1.14 on CentOS
+ARG AUTOMAKE_VER=1.14
+ARG AUTOMAKE_REPO=https://ftp.gnu.org/pub/gnu/automake/automake-${AUTOMAKE_VER}.tar.xz
+    RUN apt-get install -y -q automake
+
+# Build nginx-rtmp
+ARG NGINX_RTMP_VER=v1.2.1
+ARG NGINX_RTMP_REPO=https://github.com/arut/nginx-rtmp-module/archive/${NGINX_RTMP_VER}.tar.gz
+ARG NGINX_RTMP_PATCH_HLS=https://raw.githubusercontent.com/VCDP/CDN/master/0001-add-hevc-support-for-rtmp-and-hls.patch
+ARG NGINX_RTMP_PATCH_DASH=https://raw.githubusercontent.com/VCDP/CDN/master/0002-add-HEVC-support-for-dash.patch
+
+RUN wget -O - ${NGINX_RTMP_REPO} | tar xz && mv nginx-rtmp-module-${NGINX_RTMP_VER#v} nginx-rtmp-module && \
+    cd nginx-rtmp-module && \
+    wget -O - ${NGINX_RTMP_PATCH_HLS} | patch -p1 && \
+    wget -O - ${NGINX_RTMP_PATCH_DASH} | patch -p1
+
+
+# Build nginx & nginx-rtmp
+ARG NGINX_VER=1.14.2
+ARG NGINX_REPO=https://nginx.org/download/nginx-${NGINX_VER}.tar.gz
+
+RUN apt-get update && apt-get install -y -q --no-install-recommends libssl-dev libpcre3-dev zlib1g-dev libxslt1-dev
+
+RUN wget -O - ${NGINX_REPO} | tar xz && \
+    cd nginx-${NGINX_VER} && \
+    ./configure --prefix=/var/www --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib/x86_64-linux-gnu/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/www/log/error.log --pid-path=/var/www/nginx.pid --lock-path=/var/www/nginx.lock --http-log-path=/var/www/log/access.log --http-client-body-temp-path=/var/www/tmp/client_body --http-proxy-temp-path=/var/www/tmp/proxy --http-fastcgi-temp-path=/var/www/tmp/fastcgi --http-uwsgi-temp-path=/var/www/tmp/uwsgi --http-scgi-temp-path=/var/www/tmp/scgi --user=www-data --group=www-data --with-select_module --with-poll_module --with-threads --with-file-aio --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-http_addition_module --with-http_xslt_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_auth_request_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-stream --with-stream_ssl_module --with-stream_realip_module --with-stream_ssl_preread_module --with-pcre --add-module=../nginx-rtmp-module && \
+    make -j8 && \
+    make install DESTDIR=/home/build
+
+# NGINX Setup
+RUN mkdir -p /home/build/var/www/tmp/client_body && \
+    mkdir -p /home/build/var/www/tmp/proxy && \
+    mkdir -p /home/build/var/www/tmp/fastcgi && \
+    mkdir -p /home/build/var/www/tmp/uwsgi && \
+    mkdir -p /home/build/var/www/tmp/scgi && \
+    mkdir -p /home/build/var/www/cache && \
+    mkdir -p /home/build/var/www/html
+
+
+FROM ubuntu:18.04
+LABEL Description="This is the base image for a NGINX+RTMP service"
+LABEL Vendor="Intel Corporation"
+WORKDIR /home
+
+# Prerequisites
+RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime && \
+    DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends libxml2 libssl-dev libpcre3 zlib1g libxslt1.1 python3-tornado python3-kafka python3-bs4 python3-kazoo vim && \
+    rm -rf /var/lib/apt/lists/*
+# Install
+COPY --from=build /home/build /
+COPY *.xsl  /etc/nginx/
+COPY *.conf /etc/nginx/
+COPY *.ini  /home/
+COPY *.py   /home/
+CMD  ["/bin/bash","-c","/home/main.py&/usr/sbin/nginx"]
+VOLUME /etc/nginx /var/www/html /var/www/tmp/client_body /var/www/tmp/proxy /var/www/tmp/fastcgi /var/www/tmp/uwsgi /var/www/tmp/scgi /var/www/cache /var/www/dash /var/www/hls
+EXPOSE 8080

cdn-server/Dockerfile

@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+IMAGE="ovc_cdn_service"
+DIR=$(dirname $(readlink -f "$0"))
+
+. "${DIR}/../script/build.sh"

cdn-server/build.sh

@@ -0,0 +1,7 @@
+[run]
+timeSpan=20000
+[mode]
+srcMode=local
+[path]
+srcPath=/var/www/archive
+

cdn-server/config.ini

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+
+from tornado import ioloop, web
+from tornado.options import define, options, parse_command_line
+from playlist import PlayListHandler
+from schedule import ScheduleHandler
+
+APP = web.Application([
+    (r'/playlist', PlayListHandler),
+    (r'/schedule/.*', ScheduleHandler),
+])
+
+if __name__ == "__main__":
+    define("port", default=2222, help="the binding port", type=int)
+    define("ip", default="127.0.0.1", help="the binding ip")
+    parse_command_line()
+    print("Listening to " + options.ip + ":" + str(options.port))
+    APP.listen(options.port, address=options.ip)
+    ioloop.IOLoop.instance().start()

cdn-server/main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+
+import socket
+from kafka import KafkaProducer, KafkaConsumer, TopicPartition
+
+KAFKA_HOSTS = ["kafka:9092"]
+
+class Producer():
+    def __init__(self):
+        super(Producer, self).__init__()
+        self._client_id = socket.gethostname()
+        self._producer = None
+
+    def send(self, topic, message):
+        if not self._producer:
+            try:
+                self._producer = KafkaProducer(bootstrap_servers=KAFKA_HOSTS,
+                                               client_id=self._client_id,
+                                               api_version=(0, 10), retries=1)
+            except Exception as e:
+                print(str(e))
+                self._producer = None
+
+        if self._producer:
+            try:
+                self._producer.send(topic, message.encode('utf-8'))
+                print("send "+topic+": ")
+                print(message)
+            except Exception as e:
+                print(str(e))
+        else:
+            print("producer not available")
+
+    def flush(self):
+        if self._producer:
+            self._producer.flush()
+
+    def close(self):
+        if self._producer:
+            self.flush()
+            self._producer.close()
+
+class Consumer():
+    def __init__(self, group=None):
+        super(Consumer, self).__init__()
+        self._client_id = socket.gethostname()
+        self._group = group
+
+    def messages(self, topic, timeout=None):
+        c = KafkaConsumer(topic, bootstrap_servers=KAFKA_HOSTS, client_id=self._client_id,
+                          group_id=self._group, api_version=(0, 10))
+
+        partitions = c.partitions_for_topic(topic)
+        if not partitions:
+            raise Exception("Topic "+topic+" not exist")
+
+        timeout1 = 100 if timeout is None else timeout
+        while True:
+            partitions = c.poll(timeout1)
+            if partitions:
+                for p in partitions:
+                    for msg in partitions[p]:
+                        yield msg.value.decode('utf-8')
+            if timeout is not None:
+                yield ""
+
+        c.close()
+
+    def debug(self, topic):
+        c = KafkaConsumer(bootstrap_servers=KAFKA_HOSTS, client_id=self._client_id,
+                          group_id=None, api_version=(0, 10))
+
+        # assign/subscribe topic
+        partitions = c.partitions_for_topic(topic)
+        if not partitions:
+            raise Exception("Topic "+topic+" not exist")
+        c.assign([TopicPartition(topic, p) for p in partitions])
+
+        # seek to beginning if needed
+        c.seek_to_beginning()
+
+        # fetch messages
+        while True:
+            partitions = c.poll(100)
+            if partitions:
+                for p in partitions:
+                    for msg in partitions[p]:
+                        yield msg.value.decode('utf-8')
+            yield ""
+
+        c.close()