Dedup

dedup/parsec/gcc-hooks.bldconf

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# gcc-hooks.bldconf - configuration file for PARSEC
+
+source ${PARSECDIR}/pkgs/kernels/dedup/parsec/gcc.bldconf
+
+# Package dependencies
+build_deps="hooks ssl zlib"
+

dedup/parsec/gcc-pthreads.bldconf

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# gcc-pthreads.bldconf - file containing information necessary to build a
+#                        specific program of the PARSEC benchmark suite with
+#                        gcc and pthreads
+#
+# Copyright (C) 2007-2008 Princeton University
+# Written by Christian Bienia
+
+# Compilation and linking options.
+# This is the configuration we'll use to build the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Package dependencies
+build_deps="ssl zlib"
+
+# Environment to use for configure script and Makefile
+build_env="version=pthreads CFLAGS=\"-I${PARSECDIR}/pkgs/libs/ssl/inst/${PARSECPLAT}/include -I${PARSECDIR}/pkgs/libs/zlib/inst/${PARSECPLAT}/include ${CFLAGS}\" LDFLAGS=\"-L${PARSECDIR}/pkgs/libs/ssl/inst/${PARSECPLAT}/lib -L${PARSECDIR}/pkgs/libs/zlib/inst/${PARSECPLAT}/lib ${LDFLAGS}\""
+
+# Whether the build system supports only in-place compilation.
+# If TRUE, then all sources will be copied to the build directory before we
+# start building the package. Required for older build systems which don't
+# support VPATH.
+build_inplace="TRUE"
+
+# Arguments to pass to the configure script, if it exists
+build_conf=""

dedup/parsec/gcc-serial.bldconf

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source ${PARSECDIR}/pkgs/kernels/dedup/parsec/gcc.bldconf
+
+# Override build environment
+build_env="version=serial"
+

dedup/parsec/gcc.bldconf

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# gcc.bldconf - configuration file for PARSEC
+
+# Default parallelization is pthreads
+source ${PARSECDIR}/pkgs/kernels/dedup/parsec/gcc-pthreads.bldconf
+

dedup/parsec/icc.bldconf

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+source ${PARSECDIR}/pkgs/kernels/dedup/parsec/gcc.bldconf
+

dedup/parsec/native.runconf

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# native.runconf - file containing information necessary to run a specific
+#                  program of the PARSEC benchmark suite
+#
+# Copyright (C) 2007 Princeton University
+# Written by Christian Bienia
+
+# This is the information which we need to run the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Binary file to execute, relative to installation root
+run_exec="bin/dedup"
+
+# Arguments to use
+run_args="-c -p -v -t ${NTHREADS} -i FC-6-x86_64-disc1.iso -o output.dat.ddp"
+

dedup/parsec/simdev.runconf

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# simdev.runconf - file containing information necessary to run a specific
+#                  program of the PARSEC benchmark suite
+#
+# Copyright (C) 2007 Princeton University
+# Written by Christian Bienia
+
+# This is the information which we need to run the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Binary file to execute, relative to installation root
+run_exec="bin/dedup"
+
+# Arguments to use
+run_args="-c -p -v -t ${NTHREADS} -i hamlet.dat -o output.dat.ddp"
+

dedup/parsec/simlarge.runconf

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# simlarge.runconf - file containing information necessary to run a specific
+#                    program of the PARSEC benchmark suite
+#
+# Copyright (C) 2007 Princeton University
+# Written by Christian Bienia
+
+# This is the information which we need to run the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Binary file to execute, relative to installation root
+run_exec="bin/dedup"
+
+# Arguments to use
+run_args="-c -p -v -t ${NTHREADS} -i media.dat -o output.dat.ddp"
+

dedup/parsec/simmedium.runconf

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# simmedium.runconf - file containing information necessary to run a specific
+#                     program of the PARSEC benchmark suite
+#
+# Copyright (C) 2007 Princeton University
+# Written by Christian Bienia
+
+# This is the information which we need to run the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Binary file to execute, relative to installation root
+run_exec="bin/dedup"
+
+# Arguments to use
+run_args="-c -p -v -t ${NTHREADS} -i media.dat -o output.dat.ddp"
+

dedup/parsec/simsmall.runconf

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# simsmall.runconf - file containing information necessary to run a specific
+#                    program of the PARSEC benchmark suite
+#
+# Copyright (C) 2007 Princeton University
+# Written by Christian Bienia
+
+# This is the information which we need to run the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Binary file to execute, relative to installation root
+run_exec="bin/dedup"
+
+# Arguments to use
+run_args="-c -p -v -t ${NTHREADS} -i media.dat -o output.dat.ddp"
+

dedup/parsec/test.runconf

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# test.runconf - file containing information necessary to run a specific
+#                program of the PARSEC benchmark suite
+#
+# Copyright (C) 2007 Princeton University
+# Written by Christian Bienia
+
+# This is the information which we need to run the program. Each package of the
+# benchmark suite has such a file with information that will be added to the
+# global configuration. All variables set in the global configuration files can
+# be referenced here.
+
+
+# Binary file to execute, relative to installation root
+run_exec="bin/dedup"
+
+# Arguments to use
+run_args="-c -p -v -t ${NTHREADS} -i test.dat -o output.dat.ddp"
+

dedup/rsrc/Makefile

@@ -0,0 +1,23 @@
+include ../../parseclibs/Makefile.parsec
+NTHREADS = 1
+
+CXX:=g++
+CXXFLAGS = -O0 -g -Wno-pointer-arith -fpermissive `pkg-config --cflags raftlib` -I${PARSEC_INC}/openssl/include -DENABLE_BZIP2_COMPRESSION
+LIB = `pkg-config --libs raftlib` -L${PARSEC_LIB}/openssl/lib -lssl -lcrypto -lbz2
+
+TARGET = dedup
+
+all: $(TARGET)
+
+$(TARGET): $(TARGET).cpp
+	$(CXX) $(CXXFLAGS) -o $(TARGET) $(TARGET).cpp $(LIB)  
+
+run:
+	./$(TARGET) -c -p -v -t ${NTHREADS} -i ../inputs/hamlet.dat -o output.dat.ddp 
+
+only:
+	$(CXX) -O0 -g `pkg-config --cflags raftlib` dedup.cpp -o dedup `pkg-config --libs raftlib`
+
+clean:	
+	rm -f *.o $(TARGET)
+

dedup/rsrc/README.md

@@ -0,0 +1,14 @@
+# BUILD
+* Build RaftLib
+* Set pkg-config path
+* go to the parseclibs/openssl-src dir
+* run ```./config --prefix=`pwd`/../openssl```
+* go back to dedub dir
+* run ```make```
+
+## NOTE
+* We'll fix build system to simplify construction
+
+
+## TODO
+* Debug "GZIP Compression Not Supported" error message

dedup/rsrc/binheap.h

@@ -0,0 +1,51 @@
+/* Based on Data Structures and Algorithm Analysis in C (Second Edition)
+ * by Mark Allen Weiss.
+ *
+ * Modified by Christian Bienia, Minlan Yu.
+ */
+#ifndef _BINHEAP_H
+#define _BINHEAP_H
+
+#include "dedupdef.h"
+
+
+typedef chunk_t * HeapElementType;
+
+/* Type of a priority queue. HeapStruct is private and should not be used directly */
+struct HeapStruct {
+  int Capacity;
+  int Size;
+  HeapElementType * Elements;
+};
+
+typedef struct HeapStruct * PriorityQueue;
+
+/* Create an empty priority queue with initial capacity 'InitCapacity' */
+PriorityQueue Initialize( int InitCapacity );
+
+/* Free all data structures of PriorityQueue */
+void Destroy( PriorityQueue H );
+
+/* Delete contents of priority queue */
+void MakeEmpty( PriorityQueue H );
+
+/* Add element X to priority queue H, automatically increases capacity if queue is full */
+void Insert( HeapElementType X, PriorityQueue H );
+
+/* Return the smallest element in the priority queue (if not empty) */
+HeapElementType FindMin( PriorityQueue H );
+
+/* Delete the smallest element in the priority queue (if not empty) */
+HeapElementType DeleteMin( PriorityQueue H );
+
+/* Check whether priority queue is empty */
+int IsEmpty( PriorityQueue H );
+
+/* Check whether priority queue is full and calling Insert would result in memory reallocation */
+int IsFull( PriorityQueue H );
+
+/* Return number of elements in priority queue */
+int NumberElements( PriorityQueue H );
+
+#endif /* _BINHEAP_H */
+

dedup/rsrc/config.h

@@ -0,0 +1,17 @@
+#ifndef _CONFIG_H_
+#define _CONFIG_H_
+
+//Set to 1 to statically enable parallelization with pthreads
+//#define ENABLE_PTHREADS 1
+
+//Set to desired number of threads per queues
+//The total number of queues between two pipeline stages will be
+//greater or equal to #threads/MAX_THREADS_PER_QUEUE
+#define MAX_THREADS_PER_QUEUE 4
+
+//Set to 1 to add support with statistics collection
+//Use argument `-v' to display statistics at end of runtime
+#define ENABLE_STATISTICS 1
+
+#endif //_CONFIG_H_
+

dedup/rsrc/debug.h

@@ -0,0 +1,21 @@
+#ifndef _DEBUG_H_
+#define _DEBUG_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define TRACE(fmt, msg...) {                                               \
+       fprintf(stderr, "[%s] " fmt, __FUNCTION__, ##msg);                  \
+       }                               \
+
+#define EXIT_TRACE(fmt, msg...) {                                          \
+       TRACE(fmt, ##msg);                                                  \
+       exit(-1);                                                           \
+}
+
+#ifndef HERE
+#define HERE TRACE("file %s, line %d, func %s\n",  __FILE__, __LINE__, __FUNCTION__)
+#endif
+
+#endif //_DEBUG_H_
+

dedup/rsrc/decoder.h

@@ -0,0 +1,8 @@
+#ifndef _DECODER_H_
+#define _DECODER_H_ 1
+
+#include "dedupdef.h"
+
+void Decode(config_t * _conf);
+
+#endif /* !_DECODER_H_ */