Auto SPICE & CDF dependency download/build

Will download & build CDF and SPICE utilities as needed when the
following environment variables are not defined:
  CSPICE_INC - If not set CSPICE is downloaded
  CSPICE_LIB - If not set CSPICE is built
  CDF_INC    - If not set CDF libraries are downloaded
  CDF_LIB    - If not set CDF libs are built

For larger projects incorporating das2C, define the variables
above before calling make.

Author: cpiker

Makefile

@@ -21,8 +21,9 @@ INST_DOC=$(INST_SHARE)/doc
 endif
 
 ifeq ($(N_ARCH),)
-N_ARCH=$(shell uname).$(shell uname -m)
-N_ARCH:=$(subst /,_,$(N_ARCH))
+N_ARCH=/
+#N_ARCH=$(shell uname).$(shell uname -m)
+#N_ARCH:=$(subst /,_,$(N_ARCH))
 endif
 
 ifeq ($(INST_INC),)
@@ -37,12 +38,27 @@ ifeq ($(INST_NAT_LIB),)
 INST_NAT_LIB=$(PREFIX)/lib/$(N_ARCH)
 endif
 
+BLD_CSPICE=0
 ifeq ($(SPICE),yes)
+ifeq ($(CSPICE_INC),)
+$(info CSPICE_INC not set, download CSPICE sources if needed)
+BLD_CSPICE=1
+endif
 ifeq ($(CSPICE_LIB),)
-$(error To add spice support set CSPICE_LIB to the absolute path of your cspice.a file)
+$(info CSPICE_LIB not set, build CSPICE from source if needed)
+BLD_CSPICE=1
 endif
-ifeq ($(CSPICE_INC),)
-$(error To add spice support set CSPICE_INC to your cspice header directory)
+endif
+
+BLD_CDF=0
+ifeq ($(CDF),yes)
+ifeq ($(CDF_INC),)
+$(info CDF_INC not set, download CDF sources if needed)
+BLD_CDF=1
+endif
+ifeq ($(CDF_LIB),)
+$(info CDF_LIB not set, build CDF lib from sources if needed)
+BLD_CDF=1
 endif
 endif
 

README.md

@@ -8,23 +8,19 @@ server by a client program and a self-describing stream of data values covering
 the requested time range, at the requested time resolution, is provided in the
 response body.
 
-This package, *das2C*, provides a portable C libraries and programs for:
+This package, *das2C*, contains a portable C library  and utility
+programs that provide:
 
-  * Reading and writing dasStream versions 2.x and 3.X
+  * XML schema definitions defining das2 and das3 headers.
+  * Libs for Reading and writing dasStream versions 2.x and 3.X
   * General SI unit manipulation
   * Dataset accumulation
   * Generating spectrograms from time series data
   * Time averaging long duration spectrograms
   * Performing SPICE operations on data streams
   * Converting streams into export formats such as CSV and CDF.
 
-The core library is used by [das2py](https://github.com/das-developers/das2py) and 
-[das2dlm](https://github.com/das-developers/das2dlm). 
-
-Das2C utility programs are used by [dasFlex](https://github.com/das-developers/dasFlex) 
-web-services for server-side processing.
-
-Top level XML schema definition for das2 and das3 headers are also kept here.
+The core library is used by [das2py](https://github.com/das-developers/das2py) and [das2dlm](https://github.com/das-developers/das2dlm). The utility programs are used by [dasFlex](https://github.com/das-developers/dasFlex) web-services for server-side processing.
 
 To find out more about das2 visit https://das2.org.
 
@@ -49,8 +45,6 @@ build libdas2.3:
   * **expat** - XML Parsing library
   * **fftw3** - Fastest Fourier Transform in the West, version 3.
   * **openssl** - Secure network socket library
-  * **libcdf** - NASA Common Data Format file creation (optional)
-  * **cspice** - NAIF SPICE spacecraft position library (optional)
 
 Though package names vary from system to system, commands for installing the
 prequisites are provided below \.\.\.
@@ -70,32 +64,22 @@ $ brew install fftw
 The expat library should already be present on MacOS once the compiler install
 command `xcode-select --install` has been run.
 
-The cspice and libcdf libraries are maunal download and un-pack only.  Here's
-where I typically put them:
-```bash
-/usr/local/cspice
-/usr/local/cdf
-```
-or on Windows
-```batchfile
-C:\opt\cspice
-C:\opt\cdf
-```
+If your system already has the NAIF CSpice Toolkit and the Goddard CDF libraries
+installed you can set environment variables to incorporate them into the build,
+otherwise the Makefiles will download them automatically.
 
-## Manual Build and Install
-
-Decide where you want to install the software.  In the instructions below we've
-chosen `/usr/local` but anywhere is fine. 
+## Manual Build
 
 For POSIX compliant systems (Linux, MacOS, Android) issue the following commands
-to build, test and install the software.
+to build, and test the software.
 
-```
-$ export PREFIX=/usr/local      
-$ export N_ARCH=/               # For generic builds, omit per-OS sub-directories.
-$ make
-$ make test
-$ make install
+```bash
+$ make SPICE=yes CDF=yes
+$ make SPICE=yes CDF=yes test
+
+# To rebuild
+$ make SPICE=yes CDF=yes clean     # Removes only das2C output
+$ make SPICE=yes CDF=yes distclean # Removes CDF and SPICE libs as well
 ```
 
 For Windows systems issue the following commands in a command shell to build, test
@@ -105,32 +89,37 @@ and install the software.
 > set N_ARCH=\
 > set LIBRARY_INC=      ::location of your vcpkg installed\x64-windows-static include 
 > set LIBRARY_LIB=      ::location of your vcpkg installed\x64-windows-static lib
-> set CSPICE_INC=C:\opt\cspice\include        :: only if building with spice
-> set CSPICE_LIB=C:\opt\cspice\lib\cspice.lib :: only if building with spice
-> set CDF_INC=C:\opt\cdf\include\             :: only if building with cdf
-> set CDF_LIB=C:\opt\cdf\lib\libcdf.lib       :: only if building with cdf
-> set INSTALL_PREFIX=C:\opt   :: for example
 
-> nmake.exe /nologo /f buildfiles\Windows.mak build
-> nmake.exe /nologo /f buildfiles\Windows.mak run_test
-> nmake.exe /nologo /f buildfiles\Windows.mak install
+> vcvars.bat            ::puts nmake.exe, cl.exe, link.exe, etc. on your path
+
+> nmake.exe /nologo /f buildfiles\Windows.mak spice=yes cdf=yes build
+> nmake.exe /nologo /f buildfiles\Windows.mak spice=yes cdf=yes run_test
+```
+
+## Manual Install
 
-:: Or with CDF and SPICE support
-> nmake.exe /nologo /f buildfiles\Windows.mak SPICE=yes CDF=yes build
-> nmake.exe /nologo /f buildfiles\Windows.mak SPICE=yes CDF=yes run_test
-> nmake.exe /nologo /f buildfiles\Windows.mak SPICE=yes CDF=yes test_spice
-> nmake.exe /nologo /f buildfiles\Windows.mak SPICE=yes CDF=yes test_cdf
-> nmake.exe /nologo /f buildfiles\Windows.mak SPICE=yes CDF=yes install
+If you wish to install directly from the source tree, set an install prefix and
+then run make install.  For example on Linux:
 
+```bash
+$ make PREFIX=/usr/local SPICE=yes CDF=yes install   # adjust destination to taste
+```
+or on Windows:
+```
+> set INSTALL_PREFIX=C:\local   :: for example
+> nmake /f buildfiles\Windows.mak SPICE=yes CDF=yes install
 ```
 
-## Usage
+## Using the Libray
 
 By default all header files are copied into the subdirectory `das2` under
-`$PREFIX/include`.  When writing code that uses das2 headers add the include
-directory to the compiler command line in a manner similar to the following:
+`$PREFIX/include`.  When writing code that uses *das* headers add the top
+level include directory to the compiler command line in a manner similar to
+the following:
 ```bash
 -I $PREFIX/include 
+```
+```batchfile
 /I %PREFIX%/include
 ```
 and use the das2 subdirectory in your include statements, for example:
@@ -141,25 +130,33 @@ Common linker arguments for building libdas dependent applications follow.
 For open source programs static linking is perfectly fine:
 
 ```make
-$(PREFIX)/lib/libdas3.0.a -lexpat -lssl -lcrypto -lz -lm -lpthread                      # gnu make
+$(PREFIX)/lib/libdas.a -lexpat -lssl -lcrypto -lz -lm -lpthread                      # gnu make
 
-$(INSTALL_PREFIX)/lib/libdas3.0.lib  Advapi32.lib User32.lib Crypt32.lib ws2_32.lib     # win nmake
+$(INSTALL_PREFIX)/lib/libdas.lib  Advapi32.lib User32.lib Crypt32.lib ws2_32.lib     # win nmake
 ```
 
-For closed source applications, link against shared das2 objects (i.e. libdas3.0.so
-or das3.0dll) as required by the LGPL:
+For closed source applications, link against shared das2 objects (i.e. libdas.so.3
+or das.dll) as required by the LGPL:
 
 ```make
--L$(PREFIX)/lib -ldas2.3 -lexpat -lssl -lcrypto -lz -lm -lpthread                       # gnu make
+-L$(PREFIX)/lib -ldas -lexpat -lssl -lcrypto -lz -lm -lpthread                       # gnu make
 
 /L $(INSTALL_PREFIX)\bin das2.3.dll das2.3.lib Advapi32.lib User32.lib Crypt32.lib ws2_32.lib  # win nmake
 ```
 
-Note that on Windows, `libdas3.0lib` is the full static library but the file
-`das3.0.lib` is merely a DLL import library.
+Note that on Windows, `libdas.lib` is the full static library but the file `das.lib`
+is merely a DLL import library.
+
+
+## Using the Utility programs
+
+Most of the utility programs filters are designed to take a *das* stream
+on standard input and output a transformed stream to standard output.  The
+**das3_cdf** program
+
 
 
-## Building with XMake
+## Building with XMake (unsupported)
 
 Two build systems are provided for das2C.  Plain ole GNU Make and Microsoft NMake files, and an [xmake](https://github.com/xmake-io/xmake) file.  Since xmake is both a package manager and a build tool, you do not need to install any prerequisites to build das2C with xmake, other then your compiler and xmake itself.
 

buildfiles/Linux.mak

@@ -3,11 +3,34 @@
 MD5SUM=md5sum
 export MD5SUM
 
+BD=$(BUILD_DIR)
+
+##############################################################################
+# Upstream Source dependencies
+
+# You may already have libcdf.a and cspice.a in your build tree for other
+# parts of your project.  If so use them by setting .  If not, you can issue build depend first and dependency
+# sources will be downloaded and built.
+
+LOC_CDF_DIST:=cdf39_0-dist
+LOC_CDF_URL:=https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_0/linux/$(LOC_CDF_DIST)-cdf.tar.gz
+LOC_CSPICE_URL:=https://naif.jpl.nasa.gov/pub/naif/toolkit//C/PC_Linux_GCC_64bit/packages/cspice.tar.Z
+
+ifeq ($(BLD_CDF),1)
+CDF_INC:=$(BD)/$(LOC_CDF_DIST)/src/include
+CDF_LIB:=$(BD)/$(LOC_CDF_DIST)/src/lib/libcdf.a
+endif
+
+ifeq ($(BLD_CSPICE),1)
+CSPICE_INC:=./$(BD)/cspice/include
+CSPICE_LIB:=./$(BD)/cspice/lib/cspice.a
+endif
+
 
 ##############################################################################
 # Project definitions
 
-TARG=libdas3.0
+TARG=libdas
 
 SRCS:=das1.c array.c buffer.c builder.c cli.c codec.c credentials.c dataset.c \
 dataset_hdr2.c dataset_hdr3.c datum.c descriptor.c dft.c dimension.c dsdf.c \
@@ -47,8 +70,6 @@ ifeq ($(CDF),yes)
 UTIL_PROGS:=$(UTIL_PROGS) das3_cdf
 endif
 
-BD=$(BUILD_DIR)
-
 ##############################################################################
 # Build definitions
 
@@ -156,25 +177,32 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/%
 
 # Direct make not to nuke the intermediate .o files
 .SECONDARY: $(BUILD_OBJS) $(UTIL_OBJS)
-.PHONY: test
+.PHONY: test build
 
 ## Explicit Rules  ###########################################################
 
-build:$(BD) $(BD)/$(TARG).a $(BD)/$(TARG).so \
- $(BUILD_UTIL_PROGS) $(BUILD_TEST_PROGS)
+# Building dependencies if needed
+#ifeq ($(BLD_CSPICE)$(BLD_CDF),00)
 
-build_static:$(BD) $(BD)/$(TARG).a \
- $(BUILD_UTIL_PROGS) $(BUILD_TEST_PROGS)
+#build: $(BD)/$(TARG).a $(BD)/$(TARG).so $(BUILD_UTIL_PROGS) $(BUILD_TEST_PROGS) | $(BD)
+#build_static: $(BD) $(BD)/$(TARG).a $(BUILD_UTIL_PROGS) $(BUILD_TEST_PROGS) | $(BD)
 
-$(BD)/$(TARG).a:$(BUILD_OBJS)
-	ar rc $@ $(BUILD_OBJS)
-	
-$(BD)/$(TARG).so:$(BUILD_OBJS)
-	gcc -shared -o $@ $(BUILD_OBJS)
+#else
+
+build: build_dep $(BD)/$(TARG).a $(BD)/$(TARG).so $(BUILD_UTIL_PROGS) $(BUILD_TEST_PROGS) | $(BD)
+build_static:build_dep $(BD) $(BD)/$(TARG).a $(BUILD_UTIL_PROGS) $(BUILD_TEST_PROGS) | $(BD)
+
+#endif
 
 $(BD):
 	@if [ ! -e "$(BD)" ]; then echo mkdir $(BD); \
         mkdir $(BD); chmod g+w $(BD); fi
+
+$(BD)/$(TARG).a:$(BUILD_OBJS) | $(BD)
+	ar rc $@ $(BUILD_OBJS)
+	
+$(BD)/$(TARG).so:$(BUILD_OBJS) | $(BD)
+	gcc -shared -o $@ $(BUILD_OBJS)
 
 
 # Robert's tagged das1 reader breaks strict-aliasing expectations for C99
@@ -200,8 +228,45 @@ $(BD)/das3_cdf:utilities/das3_cdf.c $(BD)/$(TARG).a
 	@if [ "$(CDF_LIB)" = "" ] ; then echo "CDF_LIB not set"; exit 3; fi
 	$(CC) $(CFLAGS) -Wno-unused -I$(CDF_INC) -o $@ $< $(BD)/$(TARG).a $(CDF_LIB) $(LFLAGS)
 
+
+# Conditional rule
+ifeq ($(BLD_CSPICE)$(BLD_CDF),11)
+build_dep:$(CSPICE_LIB) $(CDF_LIB)
+else ifeq ($(BLD_CSPICE)$(BLD_CDF),10)
+build_dep:$(CSPICE_LIB) $(CDF_LIB)
+else ifeq ($(BLD_CSPICE)$(BLD_CDF),01)
+build_dep:$(CSPICE_LIB) $(CDF_LIB)
+else
+build_dep: 
+endif
+
+$(CSPICE_LIB): $(BD)/cspice.tar
+	cd $(BD) && tar -mxvf cspice.tar
+
+$(BD)/cspice.tar: | $(BD)
+	curl $(LOC_CSPICE_URL) > $(BD)/cspice.tar.Z
+	uncompress $(BD)/cspice.tar.Z
+
+$(CDF_LIB): $(BD)/$(LOC_CDF_DIST)
+	cd $(BUILD_DIR)/$(LOC_CDF_DIST) && $(MAKE) OS=linux ENV=gnu all
+
+$(BD)/$(LOC_CDF_DIST): | $(BD)
+	curl $(LOC_CDF_URL) > $(BD)/$(LOC_CDF_DIST)-cdf.tar.gz
+	cd $(BD) && tar -xvf $(LOC_CDF_DIST)-cdf.tar.gz
+
 # Run tests
-test: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS)
+ifeq ($(BLD_CSPICE)$(BLD_CDF),11)
+test:test_main test_spice test_cdf
+else ifeq ($(BLD_CSPICE)$(BLD_CDF),10)
+test:test_main test_spice
+else ifeq ($(BLD_CSPICE)$(BLD_CDF),01)
+test:test_main test_cdf
+else
+test:test_main
+endif
+
+
+test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS)
 	env DIFFCMD=diff test/das1_fxtime_test.sh $(BD)
 	test/das2_ascii_test1.sh $(BD)
 	test/das2_ascii_test2.sh $(BD)	
@@ -232,7 +297,7 @@ test: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS)
 	@echo "INFO: Running unit test for ragged and unique iteration, $(BD)/TestIter..."
 	$(BD)/TestIter
 	@echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv"
-	$(BD)/das3_csv < test/tracers_cdpu_status.d3b > $(BD)/tracers_cdpu_status.csv
+	$(BD)/das3_csv < test/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv
 	@echo "INFO: ==============================================="
 	@echo "INFO: All core test programs completed without errors"
 	@echo "INFO: ==============================================="		
@@ -282,10 +347,12 @@ $(INST_DOC)/das2C:$(BD)/html
 
 # Cleanup ####################################################################
 distclean:
-	if [ -d "$(BD)" ]; then rm -r $(BD); fi
+	if [ -d "$(BD)" ] ; then rm -r $(BD) ; fi
 
 clean:
-	rm -r $(BD)
+	-rm $(BD)/das* $(BD)/Test* $(BD)/libdas*
+	-rm $(BD)/*.o $(BD)/ex*
+	-rm $(BD)/cred_test*
 
 
 ## Automatic dependency tree generation for C code ###########################