feat(unigraph-dgraph): merge to upstream

Author: thesophiaxu

.github/workflows/build-dgraph.yml

@@ -0,0 +1,87 @@
+name: build-dgraph
+
+on:
+  push:
+    branches:
+      - dgraph-personal
+      
+jobs:
+  
+  build-dgraph:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2.1.5
+        with:
+          go-version: '^1.17.7' 
+
+      - name: Install dgraph dependencies
+        run: sudo apt-get update; sudo apt-get install gcc make
+
+      - name: Check Go installed
+        run: go version
+
+      - name: Checkout dgraph
+        uses: actions/checkout@v2
+        with:
+          repository: unigraph-dev/dgraph.git
+          ref: dgraph-personal
+
+      - name: Install dgraph linux
+        run: cd ./dgraph; make install; ls $(go env GOPATH)/bin;
+
+      # - name: Test install dgraph linux
+      #   run: cd ./dgraph; echo hello > $(go env GOPATH)/bin/dgraph;
+
+      - name: Correct linux build directory
+        run: mkdir $(go env GOPATH)/bin/linux_amd64/; mv $(go env GOPATH)/bin/dgraph $(go env GOPATH)/bin/linux_amd64/; ls $(go env GOPATH)/bin; ls $(go env GOPATH)/bin/linux_amd64
+
+      - name: Install dgraph macOS
+        run: GOOS=darwin make install; ls $(go env GOPATH)/bin
+
+      - name: Install dgraph macOS M1
+        run: GOOS=darwin GOARCH=arm64 make install; ls $(go env GOPATH)/bin
+
+      - name: Install dgraph windows
+        run: GOOS=windows make install; ls $(go env GOPATH)/bin
+
+      - name: Move dgraph builds to current dir
+        run: mkdir dist; mv $(go env GOPATH)/bin/linux_amd64/dgraph ./dist/dgraph_linux_amd64; mv $(go env GOPATH)/bin/darwin_amd64/dgraph ./dist/dgraph_darwin_amd64; mv $(go env GOPATH)/bin/darwin_arm64/dgraph ./dist/dgraph_darwin_arm64; mv $(go env GOPATH)/bin/windows_amd64/dgraph.exe ./dist/dgraph_windows_amd64.exe; ls .;
+
+      - name: Upload linux build as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: dgraph_linux_amd64
+          path: ./dist/dgraph_linux_amd64
+          if-no-files-found: error 
+
+      - name: Upload macOS build as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: dgraph_darwin_amd64
+          path: ./dist/dgraph_darwin_amd64
+          if-no-files-found: error 
+      
+      - name: Upload macOS M1 build as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: dgraph_darwin_arm64
+          path: ./dist/dgraph_darwin_arm64
+          if-no-files-found: error 
+
+      - name: Upload Windows build as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: dgraph_windows_amd64
+          path: ./dist/dgraph_windows_amd64.exe
+          if-no-files-found: error 
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ./dist/dgraph_linux_amd64
+            ./dist/dgraph_darwin_amd64
+            ./dist/dgraph_darwin_arm64
+            ./dist/dgraph_windows_amd64.exe
+          fail_on_unmatched_files: true

.github/workflows/golangci-lint.yml

@@ -59,7 +59,7 @@ sudo apt-get install gcc make
 Then clone the Dgraph repository and use `make install` to install the Dgraph binary to `$GOPATH/bin`.
 
 ```bash
-git clone https://github.com/dgraph-io/dgraph.git
+git clone https://github.com/unigraph-dev/dgraph.git
 cd ./dgraph
 make install
 ```

.github/workflows/label.yml

@@ -4,11 +4,14 @@
 # This gets built as part of release.sh. Must be run from /tmp/build, with the linux binaries
 # already built and placed there.
 
-FROM ubuntu:20.04
-LABEL maintainer="Dgraph Labs <contact@dgraph.io>"
+FROM golang:1.17.3-bullseye AS build
 
 RUN apt-get update && \
   apt-get install -y --no-install-recommends \
+    gcc \
+    make \
+    git \
+    bzip2 \
     ca-certificates \
     curl \
     htop \
@@ -18,9 +21,22 @@ RUN apt-get update && \
     sysstat && \
   curl -fsSL https://deb.nodesource.com/setup_14.x | bash - && \
   apt-get install -y nodejs && \
+  rm -rf /var/lib/apt/lists/* && \
+  mkdir /dgraph
+
+WORKDIR /dgraph
+COPY . .
+
+RUN make install
+
+FROM ubuntu:latest
+LABEL maintainer="Dgraph Labs <contact@dgraph.io>"
+
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends ca-certificates curl iputils-ping && \
   rm -rf /var/lib/apt/lists/*
 
-ADD linux /usr/local/bin
+COPY --from=build /go/bin/dgraph /usr/local/bin/dgraph
 
 EXPOSE 8080
 EXPOSE 9080

.github/workflows/main.yml

@@ -124,6 +124,9 @@ they form a Raft group and provide synchronous replication.
 	flag.String("export", "export", "Folder in which to store exports.")
 	flag.StringP("zero", "z", fmt.Sprintf("localhost:%d", x.PortZeroGrpc),
 		"Comma separated list of Dgraph Zero addresses of the form IP_ADDRESS:PORT.")
+	flag.Bool("expand-edge", true,
+		"Enables the expand(_predicate_) feature. This is a bit expensive for large data loads because it"+
+			" doubles the number of mutations going on in the system.")
 
 	// Useful for running multiple servers on the same machine.
 	flag.IntP("port_offset", "o", 0,
@@ -842,6 +845,7 @@ func run() {
 		ExportPath:          Alpha.Conf.GetString("export"),
 		ZeroAddr:            strings.Split(Alpha.Conf.GetString("zero"), ","),
 		Raft:                raft,
+		ExpandEdge:          Alpha.Conf.GetBool("expand-edge"),
 		WhiteListedIPRanges: ips,
 		StrictMutations:     opts.MutationsMode == worker.StrictMutations,
 		AclEnabled:          keys.AclKey != nil,

README.md

@@ -59,6 +59,7 @@ type options struct {
 	TmpDir           string
 	NumGoroutines    int
 	MapBufSize       uint64
+	ExpandEdges      bool
 	PartitionBufSize int64
 	SkipMapPhase     bool
 	CleanupTmp       bool

contrib/Dockerfile

@@ -339,6 +339,14 @@ func (m *mapper) processNQuad(nq gql.NQuad) {
 		m.addMapEntry(key, rev, shard)
 	}
 	m.addIndexMapEntries(nq, de)
+
+	if m.opt.ExpandEdges {
+		nsAttr := x.NamespaceAttr(de.Namespace, "_predicate_")
+		shard := m.state.shards.shardFor(nsAttr)
+		key = x.DataKey(nsAttr, sid)
+		pp := m.createPredicatePosting(de.Namespace, nq.Predicate)
+		m.addMapEntry(key, pp, shard)
+	}
 }
 
 func (m *mapper) uid(xid string, ns uint64) uint64 {
@@ -388,6 +396,16 @@ func (m *mapper) lookupUid(xid string, ns uint64) uint64 {
 	return uid
 }
 
+func (m *mapper) createPredicatePosting(namespace uint64, predicate string) *pb.Posting {
+	fp := farm.Fingerprint64([]byte(predicate))
+	return &pb.Posting{
+		Uid:         fp,
+		Value:       []byte(x.NamespaceAttr(namespace, predicate)),
+		ValType:     pb.Posting_DEFAULT,
+		PostingType: pb.Posting_VALUE,
+	}
+}
+
 func (m *mapper) createPostings(nq gql.NQuad,
 	de *pb.DirectedEdge) (*pb.Posting, *pb.Posting) {
 

dgraph/cmd/alpha/run.go

@@ -89,6 +89,9 @@ func init() {
 		"Number of worker threads to use. MORE THREADS LEAD TO HIGHER RAM USAGE.")
 	flag.Int64("mapoutput_mb", 2048,
 		"The estimated size of each map file output. Increasing this increases memory usage.")
+	flag.Bool("expand_edges", true,
+		"Generate edges that allow nodes to be expanded using _predicate_ or expand(...). "+
+			"Disable to increase loading speed.")
 	flag.Int64("partition_mb", 4, "Pick a partition key every N megabytes of data.")
 	flag.Bool("skip_map_phase", false,
 		"Skip the map phase (assumes that map output files already exist).")
@@ -163,6 +166,7 @@ func run() {
 		TmpDir:           Bulk.Conf.GetString("tmp"),
 		NumGoroutines:    Bulk.Conf.GetInt("num_go_routines"),
 		MapBufSize:       uint64(Bulk.Conf.GetInt("mapoutput_mb")),
+		ExpandEdges:      Bulk.Conf.GetBool("expand_edges"),
 		PartitionBufSize: int64(Bulk.Conf.GetInt("partition_mb")),
 		SkipMapPhase:     Bulk.Conf.GetBool("skip_map_phase"),
 		CleanupTmp:       Bulk.Conf.GetBool("cleanup_tmp"),