[ONOS-7808] Support P4Runtime default table entries

We achieve this by creating a special mirror to store the original
default entries as specified in the P4 program. Applications can modify
the default entry by inserting flow rules with empty selectors. When
removing such flow rule, the default table entry is restored to the
original one as stored in the mirror.

Change-Id: Ib11a7172ab56be7cbbd23022e4b62ed6b70b6eca

Author: axis0985

drivers/bmv2/src/main/resources/bmv2-drivers.xml

@@ -19,7 +19,6 @@
         <behaviour api="org.onosproject.net.behaviour.PiPipelineProgrammable"
                    impl="org.onosproject.drivers.bmv2.Bmv2PipelineProgrammable"/>
         <property name="tableDeleteBeforeUpdate">true</property>
-        <property name="supportDefaultTableEntry">false</property>
         <!--
         <behaviour api="org.onosproject.net.group.GroupProgrammable"
                    impl="org.onosproject.drivers.bmv2.Bmv2GroupProgrammable"/>
@@ -32,9 +31,6 @@
             hwVersion="BMv2 simple_switch" swVersion="Stratum" extends="stratum">
         <behaviour api="org.onosproject.net.behaviour.PiPipelineProgrammable"
                    impl="org.onosproject.drivers.bmv2.Bmv2PipelineProgrammable"/>
-        <!-- The current version of p4lang/PI used in Stratum does not
-        support reading default table entries -->
-        <property name="supportDefaultTableEntry">false</property>
     </driver>
 </drivers>
 

drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/AbstractP4RuntimePipelineProgrammable.java

@@ -16,14 +16,20 @@
 
 package org.onosproject.drivers.p4runtime;
 
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeDefaultEntryMirror;
 import org.onosproject.net.behaviour.PiPipelineProgrammable;
+import org.onosproject.net.pi.model.PiPipelineModel;
 import org.onosproject.net.pi.model.PiPipeconf;
+import org.onosproject.net.pi.runtime.PiTableEntry;
+import org.onosproject.p4runtime.api.P4RuntimeReadClient;
 import org.slf4j.Logger;
 
 import java.nio.ByteBuffer;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.SUPPORT_DEFAULT_TABLE_ENTRY;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY;
 import static java.util.concurrent.CompletableFuture.completedFuture;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -55,8 +61,9 @@ public CompletableFuture<Boolean> setPipeconf(PiPipeconf pipeconf) {
             // Hopefully the child class logged the problem.
             return completedFuture(false);
         }
-
-        return client.setPipelineConfig(p4DeviceId, pipeconf, deviceDataBuffer);
+        CompletableFuture<Boolean> pipeconfSet = client.setPipelineConfig(
+                p4DeviceId, pipeconf, deviceDataBuffer);
+        return getDefaultEntries(pipeconfSet, pipeconf);
     }
 
     @Override
@@ -70,4 +77,47 @@ public CompletableFuture<Boolean> isPipeconfSet(PiPipeconf pipeconf) {
 
     @Override
     public abstract Optional<PiPipeconf> getDefaultPipeconf();
+
+    /**
+     * Once the pipeconf is set successfully, we should store all the default entries
+     * before notify other service to prevent overwriting the default entries.
+     * Default entries may be used in P4RuntimeFlowRuleProgrammable.
+     * <p>
+     * This method returns a completable future with the result of the pipeconf set
+     * operation (which might not be true).
+     *
+     * @param pipeconfSet completable future for setting pipeconf
+     * @param pipeconf pipeconf
+     * @return completable future eventually true if the pipeconf set successfully
+     */
+    private CompletableFuture<Boolean> getDefaultEntries(CompletableFuture<Boolean> pipeconfSet, PiPipeconf pipeconf) {
+        if (!driverBoolProperty(
+                SUPPORT_DEFAULT_TABLE_ENTRY,
+                DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY)) {
+            return pipeconfSet;
+        }
+        return pipeconfSet.thenApply(setSuccess -> {
+            if (!setSuccess) {
+                return setSuccess;
+            }
+            final P4RuntimeDefaultEntryMirror mirror = handler()
+                    .get(P4RuntimeDefaultEntryMirror.class);
+
+            final PiPipelineModel pipelineModel = pipeconf.pipelineModel();
+            final P4RuntimeReadClient.ReadRequest request = client.read(
+                    p4DeviceId, pipeconf);
+            // Read default entries from all non-constant tables.
+            // Ignore constant default entries.
+            pipelineModel.tables().stream()
+                    .filter(t -> !t.isConstantTable())
+                    .forEach(t -> {
+                        if (!t.constDefaultAction().isPresent()) {
+                            request.defaultTableEntry(t.id());
+                        }
+                    });
+            final P4RuntimeReadClient.ReadResponse response = request.submitSync();
+            mirror.sync(deviceId, response.all(PiTableEntry.class));
+            return true;
+        });
+    }
 }

drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeDriverProperties.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime;
+
+/**
+ * Driver properties for P4Runtime.
+ */
+public final class P4RuntimeDriverProperties {
+
+    // hide default constructor
+    private P4RuntimeDriverProperties() {
+    }
+
+    // When updating an existing rule, if true, we issue a DELETE operation
+    // before inserting the new one, otherwise we issue a MODIFY operation. This
+    // is useful fore devices that do not support MODIFY operations for table
+    // entries.
+    public static final String DELETE_BEFORE_UPDATE = "tableDeleteBeforeUpdate";
+    public static final boolean DEFAULT_DELETE_BEFORE_UPDATE = false;
+
+    // If true, we avoid querying the device and return what's already known by
+    // the ONOS store.
+    public static final String READ_FROM_MIRROR = "tableReadFromMirror";
+    public static final boolean DEFAULT_READ_FROM_MIRROR = false;
+
+    // If true, we read counters when reading table entries (if table has
+    // counters). Otherwise, we don't.
+    public static final String SUPPORT_TABLE_COUNTERS = "supportTableCounters";
+    public static final boolean DEFAULT_SUPPORT_TABLE_COUNTERS = true;
+
+    // If true, assumes that the device returns table entry message populated
+    // with direct counter values. If false, we issue a second P4Runtime request
+    // to read the direct counter values.
+    public static final String READ_COUNTERS_WITH_TABLE_ENTRIES = "tableReadCountersWithTableEntries";
+    public static final boolean DEFAULT_READ_COUNTERS_WITH_TABLE_ENTRIES = true;
+
+    // True if target supports reading and writing table entries.
+    public static final String SUPPORT_DEFAULT_TABLE_ENTRY = "supportDefaultTableEntry";
+    public static final boolean DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY = true;
+}

drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/P4RuntimeFlowRuleProgrammable.java

@@ -21,14 +21,14 @@
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.Striped;
+import org.onosproject.drivers.p4runtime.mirror.P4RuntimeDefaultEntryMirror;
 import org.onosproject.drivers.p4runtime.mirror.P4RuntimeTableMirror;
 import org.onosproject.drivers.p4runtime.mirror.TimedEntry;
 import org.onosproject.net.flow.DefaultFlowEntry;
 import org.onosproject.net.flow.FlowEntry;
 import org.onosproject.net.flow.FlowRule;
 import org.onosproject.net.flow.FlowRuleProgrammable;
 import org.onosproject.net.pi.model.PiCounterType;
-import org.onosproject.net.pi.model.PiPipelineInterpreter;
 import org.onosproject.net.pi.model.PiPipelineModel;
 import org.onosproject.net.pi.model.PiTableId;
 import org.onosproject.net.pi.runtime.PiCounterCell;
@@ -37,6 +37,7 @@
 import org.onosproject.net.pi.runtime.PiCounterCellId;
 import org.onosproject.net.pi.runtime.PiEntityType;
 import org.onosproject.net.pi.runtime.PiHandle;
+import org.onosproject.net.pi.runtime.PiMatchKey;
 import org.onosproject.net.pi.runtime.PiTableEntry;
 import org.onosproject.net.pi.runtime.PiTableEntryHandle;
 import org.onosproject.net.pi.service.PiFlowRuleTranslator;
@@ -58,7 +59,16 @@
 import java.util.concurrent.locks.Lock;
 import java.util.stream.Collectors;
 
-import static org.onosproject.drivers.p4runtime.P4RuntimeDriverUtils.getInterpreter;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DEFAULT_DELETE_BEFORE_UPDATE;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DEFAULT_READ_COUNTERS_WITH_TABLE_ENTRIES;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DEFAULT_READ_FROM_MIRROR;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DEFAULT_SUPPORT_TABLE_COUNTERS;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.DELETE_BEFORE_UPDATE;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.READ_COUNTERS_WITH_TABLE_ENTRIES;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.READ_FROM_MIRROR;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.SUPPORT_DEFAULT_TABLE_ENTRY;
+import static org.onosproject.drivers.p4runtime.P4RuntimeDriverProperties.SUPPORT_TABLE_COUNTERS;
 import static org.onosproject.drivers.p4runtime.P4RuntimeFlowRuleProgrammable.Operation.APPLY;
 import static org.onosproject.drivers.p4runtime.P4RuntimeFlowRuleProgrammable.Operation.REMOVE;
 import static org.onosproject.net.flow.FlowEntry.FlowEntryState.ADDED;
@@ -73,40 +83,14 @@ public class P4RuntimeFlowRuleProgrammable
         extends AbstractP4RuntimeHandlerBehaviour
         implements FlowRuleProgrammable {
 
-    // When updating an existing rule, if true, we issue a DELETE operation
-    // before inserting the new one, otherwise we issue a MODIFY operation. This
-    // is useful fore devices that do not support MODIFY operations for table
-    // entries.
-    private static final String DELETE_BEFORE_UPDATE = "tableDeleteBeforeUpdate";
-    private static final boolean DEFAULT_DELETE_BEFORE_UPDATE = false;
-
-    // If true, we avoid querying the device and return what's already known by
-    // the ONOS store.
-    private static final String READ_FROM_MIRROR = "tableReadFromMirror";
-    private static final boolean DEFAULT_READ_FROM_MIRROR = false;
-
-    // If true, we read counters when reading table entries (if table has
-    // counters). Otherwise, we don't.
-    private static final String SUPPORT_TABLE_COUNTERS = "supportTableCounters";
-    private static final boolean DEFAULT_SUPPORT_TABLE_COUNTERS = true;
-
-    // If true, assumes that the device returns table entry message populated
-    // with direct counter values. If false, we issue a second P4Runtime request
-    // to read the direct counter values.
-    private static final String READ_COUNTERS_WITH_TABLE_ENTRIES = "tableReadCountersWithTableEntries";
-    private static final boolean DEFAULT_READ_COUNTERS_WITH_TABLE_ENTRIES = true;
-
-    // True if target supports reading and writing table entries.
-    private static final String SUPPORT_DEFAULT_TABLE_ENTRY = "supportDefaultTableEntry";
-    private static final boolean DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY = true;
-
     // Used to make sure concurrent calls to write flow rules are serialized so
     // that each request gets consistent access to mirror state.
     private static final Striped<Lock> WRITE_LOCKS = Striped.lock(30);
 
     private PiPipelineModel pipelineModel;
     private P4RuntimeTableMirror tableMirror;
     private PiFlowRuleTranslator translator;
+    private P4RuntimeDefaultEntryMirror defaultEntryMirror;
 
     @Override
     protected boolean setupBehaviour(String opName) {
@@ -118,6 +102,7 @@ protected boolean setupBehaviour(String opName) {
         pipelineModel = pipeconf.pipelineModel();
         tableMirror = handler().get(P4RuntimeTableMirror.class);
         translator = translationService.flowRuleTranslator();
+        defaultEntryMirror = handler().get(P4RuntimeDefaultEntryMirror.class);
         return true;
     }
 
@@ -128,7 +113,8 @@ public Collection<FlowEntry> getFlowEntries() {
             return Collections.emptyList();
         }
 
-        if (driverBoolProperty(READ_FROM_MIRROR, DEFAULT_READ_FROM_MIRROR)) {
+        if (driverBoolProperty(READ_FROM_MIRROR,
+                DEFAULT_READ_FROM_MIRROR)) {
             return getFlowEntriesFromMirror();
         }
 
@@ -237,8 +223,12 @@ private FlowEntry forgeFlowEntry(PiTableEntry entry,
                 translatedEntity = translator.lookup(handle);
         final TimedEntry<PiTableEntry> timedEntry = tableMirror.get(handle);
 
+        // A default entry might not be present in the translation store if it
+        // was not inserted by an app. No need to log.
         if (!translatedEntity.isPresent()) {
-            log.warn("Table entry handle not found in translation store: {}", handle);
+            if (!isOriginalDefaultEntry(entry)) {
+                log.warn("Table entry handle not found in translation store: {}", handle);
+            }
             return null;
         }
         if (!translatedEntity.get().translated().equals(entry)) {
@@ -390,9 +380,11 @@ private boolean appendEntryToWriteRequestOrSkip(
         final UpdateType updateType;
 
         final boolean supportDefaultEntry = driverBoolProperty(
-                SUPPORT_DEFAULT_TABLE_ENTRY, DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY);
+                SUPPORT_DEFAULT_TABLE_ENTRY,
+                DEFAULT_SUPPORT_DEFAULT_TABLE_ENTRY);
         final boolean deleteBeforeUpdate = driverBoolProperty(
-                DELETE_BEFORE_UPDATE, DEFAULT_DELETE_BEFORE_UPDATE);
+                DELETE_BEFORE_UPDATE,
+                DEFAULT_DELETE_BEFORE_UPDATE);
 
         if (driverOperation == APPLY) {
             if (piEntryOnDevice == null) {
@@ -419,8 +411,8 @@ private boolean appendEntryToWriteRequestOrSkip(
         } else {
             // REMOVE.
             if (piEntryToApply.isDefaultAction()) {
-                // Cannot remove default action. Instead we should use the
-                // original defined by the interpreter (if any).
+                // Cannot remove default action. Instead we should modify it to
+                // use the original one as specified in the P4 program.
                 final PiTableEntry originalDefaultEntry = getOriginalDefaultEntry(
                         piEntryToApply.table());
                 if (originalDefaultEntry == null) {
@@ -443,31 +435,32 @@ private boolean appendEntryToWriteRequestOrSkip(
     }
 
     private PiTableEntry getOriginalDefaultEntry(PiTableId tableId) {
-        final PiPipelineInterpreter interpreter = getInterpreter(handler());
-        if (interpreter == null) {
-            log.warn("Missing interpreter for {}, cannot get default action",
-                     deviceId);
-            return null;
-        }
-        if (!interpreter.getOriginalDefaultAction(tableId).isPresent()) {
-            log.warn("Interpreter of {} doesn't define a default action for " +
-                             "table {}, cannot produce default action entry",
-                     deviceId, tableId);
-            return null;
-        }
-        return PiTableEntry.builder()
+        final PiTableEntryHandle handle = PiTableEntry.builder()
                 .forTable(tableId)
-                .withAction(interpreter.getOriginalDefaultAction(tableId).get())
-                .build();
+                .withMatchKey(PiMatchKey.EMPTY)
+                .build()
+                .handle(deviceId);
+        final TimedEntry<PiTableEntry> originalDefaultEntry = defaultEntryMirror.get(handle);
+        if (originalDefaultEntry != null) {
+            return originalDefaultEntry.entry();
+        }
+        return null;
     }
 
     private boolean isOriginalDefaultEntry(PiTableEntry entry) {
         if (!entry.isDefaultAction()) {
             return false;
         }
         final PiTableEntry originalDefaultEntry = getOriginalDefaultEntry(entry.table());
-        return originalDefaultEntry != null &&
-                originalDefaultEntry.action().equals(entry.action());
+        if (originalDefaultEntry == null) {
+            return false;
+        }
+        // Sometimes the default action may be null
+        // e.g. In basic pipeline, the default action in wcmp_table is null
+        if (originalDefaultEntry.action() == null) {
+            return entry.action() == null;
+        }
+        return originalDefaultEntry.action().equals(entry.action());
     }
 
     private Map<PiTableEntryHandle, PiCounterCellData> readEntryCounters(

drivers/p4runtime/src/main/java/org/onosproject/drivers/p4runtime/mirror/DistributedP4RuntimeDefaultEntryMirror.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.drivers.p4runtime.mirror;
+
+import org.onosproject.net.pi.runtime.PiEntityType;
+import org.onosproject.net.pi.runtime.PiTableEntry;
+import org.onosproject.net.pi.runtime.PiTableEntryHandle;
+import org.osgi.service.component.annotations.Component;
+
+/**
+ * Distributed implementation of a P4Runtime default entry mirror.
+ */
+@Component(immediate = true, service = P4RuntimeDefaultEntryMirror.class)
+public final class DistributedP4RuntimeDefaultEntryMirror
+        extends AbstractDistributedP4RuntimeMirror
+                        <PiTableEntryHandle, PiTableEntry>
+        implements P4RuntimeDefaultEntryMirror {
+
+    public DistributedP4RuntimeDefaultEntryMirror() {
+        super(PiEntityType.TABLE_ENTRY);
+    }
+}