Merge pull request #758 from qubic/release/release-278

* Adds Oracles to the core
* Remove dependency from QBAY and NOSTROMO to QUOTTERY datetime
* Solves issue with Unions for non-MSVC
* Fix revenue of computors to 100% due to potential mid-epoch updates
* Increase ADDITION_SOLUTION_THRESHOLD_DEFAULT to 74196

Author: fnordspace

doc/contracts_proposals.md

@@ -120,7 +120,7 @@ struct QUTIL
         // Your code should set the state variable that the proposal is about to the accepted value.
         // This can be done as in this example taken from QUTIL:
 
-        switch (input.proposal.variableOptions.variable)
+        switch (input.proposal.data.variableOptions.variable)
         {
         case 0:
             state.smt1InvocationFee = input.acceptedValue;
@@ -348,7 +348,7 @@ PRIVATE_PROCEDURE_WITH_LOCALS(FinalizeShareholderStateVarProposals)
             if (locals.p.acceptedOption <= 0)
                 continue;
 
-            locals.p.acceptedValue = locals.p.proposal.variableOptions.value;
+            locals.p.acceptedValue = locals.p.proposal.data.variableOptions.value;
 
             CALL(FinalizeShareholderProposalSetStateVar, locals.p, output);
         }
@@ -376,8 +376,8 @@ PUBLIC_PROCEDURE(SetShareholderProposal)
 {
     // check proposal input and fees
     if (qpi.invocationReward() < setProposalFeeVarOrValue || (input.epoch
-        && (input.type != ProposalTypes::VariableYesNo || input.variableOptions.variable >= numFeeStateVariables
-            || input.variableOptions.value < 0)))
+        && (input.type != ProposalTypes::VariableYesNo || input.data.variableOptions.variable >= numFeeStateVariables
+            || input.data.variableOptions.value < 0)))
     {
         // error -> reimburse invocation reward
         qpi.transfer(qpi.invocator(), qpi.invocationReward());

doc/protocol.md

@@ -13,6 +13,24 @@ The network message types are defined in a single `enum` in [src/network_message
 The type number is the identifier used in `RequestResponseHeader` (defined in `header.h`).
 The type number is usually available from the network message type via the `static constexpr unsigned char type()` method.
 
+
+## List of special transaction types
+
+Transactions with destination `NULL_ID` (32 bytes all-zero) are handled by the system.
+The following transaction types (`tx->inputType`) are defined:
+
+- Vote counter transaction, type 1, defined in `src/vote_counter.h`.
+- `MiningSolutionTransaction`, type 2, defined in `src/mining/mining.h`.
+- `FileHeaderTransaction`, type 3, defined in `src/files/files.h`.
+- `FileFragmentTransactionPrefix`, type 4, defined in `src/files/files.h`.
+- `FileTrailerTransaction`, type 5, defined in `src/files/files.h`.
+- `OracleReplyCommitTransactionPrefix`, type 6, defined in `src/oracle_core/oracle_transactions.h`.
+- `OracleReplyRevealTransactionPrefix`, type 7, defined in `src/oracle_core/oracle_transactions.h`.
+- `CustomMiningSolutionTransaction`, type 8, defined in `src/mining/mining.h`.
+- `ExecutionFeeReportTransactionPrefix`, type 9, defined in `src/network_messages/execution_fees.h`.
+- `OracleUserQueryTransactionPrefix`, type 10, defined in `src/oracle_core/oracle_transactions.h`.
+
+
 ## Peer Sharing
 
 Peers are identified by IPv4 addresses in context of peer sharing.

src/Qubic.vcxproj

@@ -62,6 +62,7 @@
     <ClInclude Include="contract_core\qpi_collection_impl.h" />
     <ClInclude Include="contract_core\qpi_ipo_impl.h" />
     <ClInclude Include="contract_core\qpi_mining_impl.h" />
+    <ClInclude Include="contract_core\qpi_oracle_impl.h" />
     <ClInclude Include="contract_core\qpi_spectrum_impl.h" />
     <ClInclude Include="contract_core\qpi_system_impl.h" />
     <ClInclude Include="contract_core\qpi_hash_map_impl.h" />
@@ -92,13 +93,20 @@
     <ClInclude Include="network_messages\execution_fees.h" />
     <ClInclude Include="network_messages\header.h" />
     <ClInclude Include="network_messages\logging.h" />
+    <ClInclude Include="network_messages\oracles.h" />
     <ClInclude Include="network_messages\public_peers.h" />
     <ClInclude Include="network_messages\special_command.h" />
     <ClInclude Include="network_messages\system_info.h" />
     <ClInclude Include="network_messages\tick.h" />
     <ClInclude Include="network_messages\transactions.h" />
-    <ClInclude Include="oracles\oracle_machines.h" />
-    <ClInclude Include="oracles\Price.h" />
+    <ClInclude Include="oracle_core\core_om_network_messages.h" />
+    <ClInclude Include="oracle_core\net_msg_impl.h" />
+    <ClInclude Include="oracle_core\oracle_engine.h" />
+    <ClInclude Include="oracle_core\oracle_interfaces_def.h" />
+    <ClInclude Include="oracle_core\oracle_transactions.h" />
+    <ClInclude Include="oracle_core\snapshot_files.h" />
+    <ClInclude Include="oracle_interfaces\Mock.h" />
+    <ClInclude Include="oracle_interfaces\Price.h" />
     <ClInclude Include="platform\assert.h" />
     <ClInclude Include="platform\concurrency.h" />
     <ClInclude Include="four_q.h" />
@@ -110,6 +118,7 @@
     <ClInclude Include="platform\file_io.h" />
     <ClInclude Include="platform\console_logging.h" />
     <ClInclude Include="platform\common_types.h" />
+    <ClInclude Include="platform\memory_util.h" />
     <ClInclude Include="platform\profiling.h" />
     <ClInclude Include="platform\quorum_value.h" />
     <ClInclude Include="platform\random.h" />
@@ -123,7 +132,6 @@
     <ClInclude Include="score.h" />
     <ClInclude Include="platform\m256.h" />
     <ClInclude Include="platform\memory.h" />
-    <ClInclude Include="platform\memory-util.h" />
     <ClInclude Include="score_cache.h" />
     <ClInclude Include="spectrum\special_entities.h" />
     <ClInclude Include="spectrum\spectrum.h" />

src/Qubic.vcxproj.filters

@@ -182,12 +182,6 @@
     <ClInclude Include="contracts\ComputorControlledFund.h">
       <Filter>contracts</Filter>
     </ClInclude>
-    <ClInclude Include="oracles\Price.h">
-      <Filter>oracles</Filter>
-    </ClInclude>
-    <ClInclude Include="oracles\oracle_machines.h">
-      <Filter>oracles</Filter>
-    </ClInclude>
     <ClInclude Include="files\files.h">
       <Filter>files</Filter>
     </ClInclude>
@@ -230,7 +224,6 @@
     <ClInclude Include="contracts\TestExampleB.h">
       <Filter>contracts</Filter>
     </ClInclude>
-    <ClInclude Include="platform\memory-util.h" />
     <ClInclude Include="ticking\ticking.h">
       <Filter>ticking</Filter>
     </ClInclude>
@@ -315,12 +308,42 @@
     <ClInclude Include="contract_core\pre_qpi_def.h">
       <Filter>contract_core</Filter>
     </ClInclude>
+    <ClInclude Include="oracle_core\oracle_engine.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_core\core_om_network_messages.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_core\oracle_interfaces_def.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_core\oracle_transactions.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_interfaces\Price.h">
+      <Filter>oracle_interfaces</Filter>
+    </ClInclude>
+    <ClInclude Include="platform\memory_util.h">
+      <Filter>platform</Filter>
+    </ClInclude>
+    <ClInclude Include="contract_core\qpi_oracle_impl.h">
+      <Filter>contract_core</Filter>
+    </ClInclude>
     <ClInclude Include="network_messages\network_message_type.h">
       <Filter>network_messages</Filter>
     </ClInclude>
+    <ClInclude Include="network_messages\oracles.h">
+      <Filter>network_messages</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_core\net_msg_impl.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
     <ClInclude Include="contract_core\execution_time_accumulator.h">
       <Filter>contract_core</Filter>
     </ClInclude>
+    <ClInclude Include="oracle_interfaces\Mock.h">
+      <Filter>oracle_interfaces</Filter>
+    </ClInclude>
     <ClInclude Include="mining\score_common.h">
       <Filter>mining</Filter>
     </ClInclude>
@@ -336,6 +359,12 @@
     <ClInclude Include="contracts\qRWA.h">
       <Filter>contracts</Filter>
     </ClInclude>
+    <ClInclude Include="contracts\SupplyWatcher_old.h">
+      <Filter>contracts</Filter>
+    </ClInclude>
+    <ClInclude Include="oracle_core\snapshot_files.h">
+      <Filter>oracle_core</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Filter Include="platform">
@@ -356,9 +385,6 @@
     <Filter Include="addons">
       <UniqueIdentifier>{d409258b-a63c-443f-8c7a-918c14f192a6}</UniqueIdentifier>
     </Filter>
-    <Filter Include="oracles">
-      <UniqueIdentifier>{74ae36a2-d776-43e9-a6b9-f41412f5fdd2}</UniqueIdentifier>
-    </Filter>
     <Filter Include="files">
       <UniqueIdentifier>{ed6ee181-230c-46ac-91bd-234b9081e36f}</UniqueIdentifier>
     </Filter>
@@ -377,10 +403,16 @@
     <Filter Include="spectrum">
       <UniqueIdentifier>{2f27aed7-b9b8-4b8c-a70a-90f18385b167}</UniqueIdentifier>
     </Filter>
+    <Filter Include="oracle_interfaces">
+      <UniqueIdentifier>{ec5bb5e9-cc03-4d2f-bbe4-40d60c3df1c8}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="oracle_core">
+      <UniqueIdentifier>{a42dd6f2-617c-4fbc-b3ec-f51975e5c447}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <MASM Include="platform\custom_stack.asm">
       <Filter>platform</Filter>
     </MASM>
   </ItemGroup>
-</Project>
+</Project>

src/contract_core/contract_def.h

@@ -15,6 +15,9 @@
 #include "contracts/qpi.h"
 #include "qpi_proposal_voting.h"
 
+// make interfaces to oracles available for all contracts
+#include "oracle_core/oracle_interfaces_def.h"
+
 #define QX_CONTRACT_INDEX 1
 #define CONTRACT_INDEX QX_CONTRACT_INDEX
 #define CONTRACT_STATE_TYPE QX
@@ -479,10 +482,55 @@ static void initializeContracts()
     REGISTER_CONTRACT_FUNCTIONS_AND_PROCEDURES(TESTEXD);
 
     // fill execution fee reserves for test contracts
-    setContractFeeReserve(TESTEXA_CONTRACT_INDEX, 10000);
-    setContractFeeReserve(TESTEXB_CONTRACT_INDEX, 10000);
-    setContractFeeReserve(TESTEXC_CONTRACT_INDEX, 10000);
-    setContractFeeReserve(TESTEXD_CONTRACT_INDEX, 10000);
+    setContractFeeReserve(TESTEXA_CONTRACT_INDEX, 100000000000);
+    setContractFeeReserve(TESTEXB_CONTRACT_INDEX, 100000000000);
+    setContractFeeReserve(TESTEXC_CONTRACT_INDEX, 100000000000);
+    setContractFeeReserve(TESTEXD_CONTRACT_INDEX, 100000000000);
 #endif
 }
 
+// Class for registering and looking up user procedures independently of input type, for example for notifications
+class UserProcedureRegistry
+{
+public:
+    struct UserProcedureData
+    {
+        USER_PROCEDURE procedure;
+        unsigned int contractIndex;
+        unsigned int localsSize;
+        unsigned short inputSize;
+        unsigned short outputSize;
+    };
+
+    void init()
+    {
+        setMemory(*this, 0);
+    }
+
+    bool add(unsigned int procedureId, const UserProcedureData& data)
+    {
+        const unsigned int cnt = (unsigned int)idToIndex.population();
+        if (cnt >= idToIndex.capacity())
+            return false;
+
+        copyMemory(userProcData[cnt], data);
+        idToIndex.set(procedureId, cnt);
+
+        return true;
+    }
+
+    const UserProcedureData* get(unsigned int procedureId) const
+    {
+        unsigned int idx;
+        if (!idToIndex.get(procedureId, idx))
+            return nullptr;
+        return userProcData + idx;
+    }
+
+protected:
+    UserProcedureData userProcData[MAX_CONTRACT_PROCEDURES_REGISTERED];
+    QPI::HashMap<unsigned int, unsigned int, MAX_CONTRACT_PROCEDURES_REGISTERED> idToIndex;
+};
+
+// For registering and looking up user procedures independently of input type (for notifications), initialized by initContractExec()
+GLOBAL_VAR_DECL UserProcedureRegistry* userProcedureRegistry GLOBAL_VAR_INIT(nullptr);

src/contract_core/contract_exec.h

@@ -192,6 +192,12 @@ static bool initContractExec()
     if (!contractActionTracker.allocBuffer())
         return false;
 
+    if (!allocPoolWithErrorLog(L"userProcedureRegistry", sizeof(*userProcedureRegistry), (void**)&userProcedureRegistry, __LINE__))
+    {
+        return false;
+    }
+    userProcedureRegistry->init();
+
     return true;
 }
 
@@ -218,6 +224,9 @@ static void deinitContractExec()
         freePool(contractStateChangeFlags);
     }
 
+    if (userProcedureRegistry)
+        freePool(userProcedureRegistry);
+
     contractActionTracker.freeBuffer();
 }
 
@@ -917,6 +926,16 @@ void QPI::QpiContextForInit::__registerUserProcedure(USER_PROCEDURE userProcedur
     contractUserProcedureLocalsSizes[_currentContractIndex][inputType] = localsSize;
 }
 
+void QPI::QpiContextForInit::__registerUserProcedureNotification(USER_PROCEDURE userProcedure, unsigned int procedureId, unsigned short inputSize, unsigned short outputSize, unsigned int localsSize) const
+{
+    ASSERT(userProcedureRegistry);
+    if (!userProcedureRegistry->add(procedureId, { userProcedure, _currentContractIndex, localsSize, inputSize, outputSize }))
+    {
+#if !defined(NDEBUG)
+        addDebugMessage(L"__registerUserProcedureNotification() failed. You should increase MAX_CONTRACT_PROCEDURES_REGISTERED.");
+#endif
+    }
+}
 
 
 // QPI context used to call contract system procedure from qubic core (contract processor)
@@ -1266,15 +1285,6 @@ struct QpiContextUserFunctionCall : public QPI::QpiContextFunctionCall
 };
 
 
-struct UserProcedureNotification
-{
-    unsigned int contractIndex;
-    USER_PROCEDURE procedure;
-    const void* inputPtr;
-    unsigned short inputSize;
-    unsigned int localsSize;
-};
-
 // QPI context used to call contract user procedure as a notification from qubic core (contract processor).
 // This means, it isn't triggered by a transaction, but following an event after having setup the notification
 // callback in the contract code.
@@ -1283,16 +1293,16 @@ struct UserProcedureNotification
 // The procedure pointer, the expected inputSize, and the expected localsSize, which are passed via
 // UserProcedureNotification, must be consistent. The code using notifications is responible for ensuring that.
 // Use cases:
-// - oracle notifications (managed by oracleEngine)
+// - oracle notifications (managed by oracleEngine and userProcedureRegistry)
 struct QpiContextUserProcedureNotificationCall : public QPI::QpiContextProcedureCall
 {
-    QpiContextUserProcedureNotificationCall(const UserProcedureNotification& notification) : QPI::QpiContextProcedureCall(notif.contractIndex, NULL_ID, 0, USER_PROCEDURE_NOTIFICATION_CALL),  notif(notification)
+    QpiContextUserProcedureNotificationCall(const UserProcedureRegistry::UserProcedureData& notification) : QPI::QpiContextProcedureCall(notification.contractIndex, NULL_ID, 0, USER_PROCEDURE_NOTIFICATION_CALL),  notif(notification)
     {
         contractActionTracker.init();
     }
 
     // Run user procedure notification
-    void call()
+    void call(const void* inputPtr)
     {
         ASSERT(_currentContractIndex < contractCount);
 
@@ -1330,7 +1340,7 @@ struct QpiContextUserProcedureNotificationCall : public QPI::QpiContextProcedure
             __qpiAbort(ContractErrorAllocInputOutputFailed);
         }
         char* locals = input + notif.inputSize;
-        copyMem(input, notif.inputPtr, notif.inputSize);
+        copyMem(input, inputPtr, notif.inputSize);
         setMem(locals, notif.localsSize, 0);
 
         // call user procedure
@@ -1353,5 +1363,5 @@ struct QpiContextUserProcedureNotificationCall : public QPI::QpiContextProcedure
     }
 
 private:
-    const UserProcedureNotification& notif;
+    const UserProcedureRegistry::UserProcedureData& notif;
 };

src/contract_core/execution_time_accumulator.h

@@ -59,7 +59,7 @@ class ExecutionTimeAccumulator
             math_lib::sadd(contractExecutionTimePerPhase[contractExecutionTimeActiveArrayIndex][contractIndex], timeMicroSeconds);
         RELEASE(lock);
 
-#if !defined(NDEBUG) && !defined(NO_UEFI)
+#if !defined(NDEBUG) && !defined(NO_UEFI) && 0
         CHAR16 dbgMsgBuf[128];
         setText(dbgMsgBuf, L"Execution time added for contract ");
         appendNumber(dbgMsgBuf, contractIndex, FALSE);

src/contract_core/pre_qpi_def.h

@@ -27,6 +27,9 @@ constexpr unsigned short MAX_NESTED_CONTRACT_CALLS = 10;
 // Size of the contract action tracker, limits the number of transfers that one contract call can execute.
 constexpr unsigned long long CONTRACT_ACTION_TRACKER_SIZE = 16 * 1024 * 1024;
 
+// Maximum number of contract procedures that may be registered, e.g. for user procedure notifications
+constexpr unsigned int MAX_CONTRACT_PROCEDURES_REGISTERED = 16 * 1024;
+
 
 static void __beginFunctionOrProcedure(const unsigned int); // TODO: more human-readable form of function ID?
 static void __endFunctionOrProcedure(const unsigned int);