From 1189caf306075ed84555d154427a07b951a45be0 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Tue, 15 Sep 2020 09:34:34 +0800 Subject: [PATCH 01/29] Implement action versioning --- ...esign_document_for_entities_db_v2.1.0.json | 5 +- .../openwhisk/core/connector/Message.scala | 1 + .../database/RemoteCacheInvalidation.scala | 18 +- .../apache/openwhisk/core/entity/SemVer.scala | 2 +- .../openwhisk/core/entity/WhiskAction.scala | 138 +++++++++-- .../openwhisk/core/entity/WhiskEntity.scala | 2 +- .../openwhisk/core/controller/Actions.scala | 113 +++++++-- .../openwhisk/core/controller/ApiUtils.scala | 2 +- .../openwhisk/core/controller/Rules.scala | 70 ++++-- .../openwhisk/core/controller/Triggers.scala | 26 +- .../controller/actions/PrimitiveActions.scala | 1 + .../loadBalancer/InvokerSupervision.scala | 1 + .../core/invoker/InvokerReactive.scala | 7 +- .../test/scala/common/WskCliOperations.scala | 1 + .../src/test/scala/common/WskOperations.scala | 1 + .../scala/common/rest/WskRestOperations.scala | 8 +- .../DockerToActivationLogStoreTests.scala | 6 +- .../test/ContainerPoolTests.scala | 16 +- .../test/ContainerProxyTests.scala | 15 +- .../controller/test/ActionsApiTests.scala | 232 +++++++++++++----- .../test/EntitlementProviderTests.scala | 6 +- .../controller/test/PackagesApiTests.scala | 6 +- .../controller/test/SequenceApiTests.scala | 14 +- .../core/entity/test/SchemaTests.scala | 22 +- .../test/InvokerSupervisionTests.scala | 1 + .../ShardingContainerPoolBalancerTests.scala | 23 +- .../scala/system/basic/WskActionTests.scala | 32 +++ .../system/basic/WskRestBasicTests.scala | 13 - 28 files changed, 579 insertions(+), 203 deletions(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index 34d52f8f052..ccac41a016f 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -21,6 +21,9 @@ "triggers": { "map": "function (doc) {\n var PATHSEP = \"/\";\n var isTrigger = function (doc) { return (doc.exec === undefined && doc.binding === undefined && doc.parameters !== undefined) };\n if (isTrigger(doc)) try {\n var ns = doc.namespace.split(PATHSEP);\n var root = ns[0];\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations,\n updated: doc.updated\n };\n emit([doc.namespace, doc.updated], value);\n if (root !== doc.namespace) {\n emit([root, doc.updated], value);\n }\n } catch (e) {}\n}", "reduce": "_count" + }, + "action-versions": { + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n id: doc._id,\n version: doc.version,\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } -} \ No newline at end of file +} diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala index ba05c17964c..2e28d3d141c 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala @@ -53,6 +53,7 @@ case class ActivationMessage(override val transid: TransactionId, revision: DocRevision, user: Identity, activationId: ActivationId, + actionId: DocId, rootControllerIndex: ControllerInstanceId, blocking: Boolean, content: Option[JsObject], diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala index c5b5e019b3a..b081dbfe07c 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala @@ -32,13 +32,16 @@ import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} import org.apache.openwhisk.core.connector.Message import org.apache.openwhisk.core.connector.MessageFeed import org.apache.openwhisk.core.connector.MessagingProvider -import org.apache.openwhisk.core.entity.CacheKey -import org.apache.openwhisk.core.entity.ControllerInstanceId -import org.apache.openwhisk.core.entity.WhiskAction -import org.apache.openwhisk.core.entity.WhiskActionMetaData -import org.apache.openwhisk.core.entity.WhiskPackage -import org.apache.openwhisk.core.entity.WhiskRule -import org.apache.openwhisk.core.entity.WhiskTrigger +import org.apache.openwhisk.core.entity.{ + CacheKey, + ControllerInstanceId, + WhiskAction, + WhiskActionMetaData, + WhiskActionVersionList, + WhiskPackage, + WhiskRule, + WhiskTrigger +} import org.apache.openwhisk.spi.SpiLoader import pureconfig._ @@ -92,6 +95,7 @@ class RemoteCacheInvalidation(config: WhiskConfig, component: String, instance: WhiskPackage.removeId(msg.key) WhiskRule.removeId(msg.key) WhiskTrigger.removeId(msg.key) + WhiskActionVersionList.removeId(msg.key) } } case Failure(t) => logging.error(this, s"failed processing message: $raw with $t") diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala index d9c5efd3034..7cbf1c802b0 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala @@ -78,7 +78,7 @@ protected[core] object SemVer { * @return SemVer instance * @thrown IllegalArgumentException if string is not a valid semantic version */ - protected[entity] def apply(str: String): SemVer = { + protected[core] def apply(str: String): SemVer = { try { val parts = if (str != null && str.nonEmpty) str.split('.') else Array[String]() val major = if (parts.size >= 1) parts(0).toInt else 0 diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index a51e5527889..f0fbf6fd02a 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -29,10 +29,14 @@ import scala.concurrent.Future import scala.util.{Failure, Success, Try} import spray.json._ import spray.json.DefaultJsonProtocol._ -import org.apache.openwhisk.common.TransactionId -import org.apache.openwhisk.core.database.ArtifactStore -import org.apache.openwhisk.core.database.DocumentFactory -import org.apache.openwhisk.core.database.CacheChangeNotification +import org.apache.openwhisk.common.{Logging, TransactionId} +import org.apache.openwhisk.core.database.{ + ArtifactStore, + CacheChangeNotification, + DocumentFactory, + MultipleReadersSingleWriterCache, + StaleParameter +} import org.apache.openwhisk.core.entity.Attachments._ import org.apache.openwhisk.core.entity.types.EntityStore @@ -106,8 +110,10 @@ abstract class WhiskActionLike(override val name: EntityName) extends WhiskEntit "annotations" -> annotations.toJson) } -abstract class WhiskActionLikeMetaData(override val name: EntityName) extends WhiskActionLike(name) { +abstract class WhiskActionLikeMetaData(override val name: EntityName, val docId: DocId) extends WhiskActionLike(name) { override def exec: ExecMetaDataBase + + override def docid = docId } /** @@ -143,6 +149,8 @@ case class WhiskAction(namespace: EntityPath, require(exec != null, "exec undefined") require(limits != null, "limits undefined") + override def docid = DocId(fullyQualifiedName(true).asString) + /** * Merges parameters (usually from package) with existing action parameters. * Existing parameters supersede those in p. @@ -173,7 +181,7 @@ case class WhiskAction(namespace: EntityPath, def toExecutableWhiskAction: Option[ExecutableWhiskAction] = exec match { case codeExec: CodeExec[_] => Some( - ExecutableWhiskAction(namespace, name, codeExec, parameters, limits, version, publish, annotations) + ExecutableWhiskAction(namespace, name, docid, codeExec, parameters, limits, version, publish, annotations) .revision[ExecutableWhiskAction](rev)) case _ => None } @@ -198,6 +206,7 @@ case class WhiskAction(namespace: EntityPath, @throws[IllegalArgumentException] case class WhiskActionMetaData(namespace: EntityPath, override val name: EntityName, + override val docId: DocId, exec: ExecMetaDataBase, parameters: Parameters = Parameters(), limits: ActionLimits = ActionLimits(), @@ -206,7 +215,7 @@ case class WhiskActionMetaData(namespace: EntityPath, annotations: Parameters = Parameters(), override val updated: Instant = WhiskEntity.currentMillis(), binding: Option[EntityPath] = None) - extends WhiskActionLikeMetaData(name) { + extends WhiskActionLikeMetaData(name, docId) { require(exec != null, "exec undefined") require(limits != null, "limits undefined") @@ -238,6 +247,7 @@ case class WhiskActionMetaData(namespace: EntityPath, ExecutableWhiskActionMetaData( namespace, name, + docId, execMetaData, parameters, limits, @@ -276,6 +286,7 @@ case class WhiskActionMetaData(namespace: EntityPath, @throws[IllegalArgumentException] case class ExecutableWhiskAction(namespace: EntityPath, override val name: EntityName, + docId: DocId, exec: CodeExec[_], parameters: Parameters = Parameters(), limits: ActionLimits = ActionLimits(), @@ -322,6 +333,7 @@ case class ExecutableWhiskAction(namespace: EntityPath, @throws[IllegalArgumentException] case class ExecutableWhiskActionMetaData(namespace: EntityPath, override val name: EntityName, + override val docId: DocId, exec: ExecMetaData, parameters: Parameters = Parameters(), limits: ActionLimits = ActionLimits(), @@ -329,13 +341,13 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, publish: Boolean = false, annotations: Parameters = Parameters(), binding: Option[EntityPath] = None) - extends WhiskActionLikeMetaData(name) { + extends WhiskActionLikeMetaData(name, docId) { require(exec != null, "exec undefined") require(limits != null, "limits undefined") def toWhiskAction = - WhiskActionMetaData(namespace, name, exec, parameters, limits, version, publish, annotations, updated) + WhiskActionMetaData(namespace, name, docId, exec, parameters, limits, version, publish, annotations, updated) .revision[WhiskActionMetaData](rev) /** @@ -346,6 +358,74 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, } +case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer) + +object WhiskActionVersion { + val serdes = jsonFormat4(WhiskActionVersion.apply) +} + +case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versions: Map[SemVer, String]) { + def matchedDocId(version: Option[SemVer]): Option[DocId] = { + version match { + case Some(ver) => + versions.get(ver).map(DocId(_)) + case None if versions.nonEmpty => + Some(DocId(versions.maxBy(_._1.toString)._2)) + case _ => + None + } + } +} + +object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActionVersionList, DocInfo] { + lazy val viewName = WhiskQueries.entitiesView(collection = "action-versions").name + + def cacheKey(action: FullyQualifiedEntityName): CacheKey = { + CacheKey(action.fullPath.asString) + } + + def get(action: FullyQualifiedEntityName, datastore: EntityStore)( + implicit transId: TransactionId): Future[WhiskActionVersionList] = { + implicit val logger: Logging = datastore.logging + implicit val ec = datastore.executionContext + + val key = List(action.fullPath.asString) + cacheLookup( + cacheKey(action), + datastore + .query( + viewName, + startKey = key, + endKey = key, + skip = 0, + limit = 0, + includeDocs = false, + descending = false, + reduce = false, + stale = StaleParameter.No) + .map { result => + val values = result.map { row => + row.fields("value").asJsObject() + } + val mappings = values + .map(WhiskActionVersion.serdes.read(_)) + .map { actionVersion => + (actionVersion.version, actionVersion.id) + } + .toMap + WhiskActionVersionList(action.namespace.toPath, action.name, mappings) + }) + } + + // delete cache + def deleteCache(action: FullyQualifiedEntityName)(implicit transId: TransactionId, + ec: ExecutionContext, + logger: Logging, + notifier: Option[CacheChangeNotification]) = { + cacheInvalidate(cacheKey(action), Future.successful(())) + } +} + object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[WhiskAction] with DefaultJsonProtocol { import WhiskActivation.instantSerdes @@ -415,7 +495,11 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ old) } } match { - case Success(f) => f + case Success(f) => + implicit val ec = db.executionContext + implicit val logger = db.logging + WhiskActionVersionList.deleteCache(doc.fullyQualifiedName(false)) + f case Failure(f) => Future.failed(f) } } @@ -537,14 +621,18 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ * If it's the actual package, use its name directly as the package path name. * While traversing the package bindings, merge the parameters. */ - def resolveActionAndMergeParameters(entityStore: EntityStore, fullyQualifiedName: FullyQualifiedEntityName)( - implicit ec: ExecutionContext, - transid: TransactionId): Future[WhiskAction] = { + def resolveActionAndMergeParameters( + entityStore: EntityStore, + fullyQualifiedName: FullyQualifiedEntityName, + version: Option[SemVer] = None)(implicit ec: ExecutionContext, transid: TransactionId): Future[WhiskAction] = { // first check that there is a package to be resolved val entityPath = fullyQualifiedName.path if (entityPath.defaultPackage) { // this is the default package, nothing to resolve - WhiskAction.get(entityStore, fullyQualifiedName.toDocId) + WhiskActionVersionList.get(fullyQualifiedName, entityStore).flatMap { result => + val docId = result.matchedDocId(version).getOrElse(fullyQualifiedName.toDocId) + WhiskAction.get(entityStore, docId) + } } else { // there is a package to be resolved val pkgDocid = fullyQualifiedName.path.toDocId @@ -553,8 +641,12 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ wp flatMap { resolvedPkg => // fully resolved name for the action val fqnAction = resolvedPkg.fullyQualifiedName(withVersion = false).add(actionName) + val action = WhiskActionVersionList.get(fqnAction, entityStore).flatMap { result => + val docId = result.matchedDocId(version).getOrElse(fqnAction.toDocId) + WhiskAction.get(entityStore, docId) + } // get the whisk action associate with it and inherit the parameters from the package/binding - WhiskAction.get(entityStore, fqnAction.toDocId) map { + action map { _.inherit(resolvedPkg.parameters) } } @@ -576,6 +668,7 @@ object WhiskActionMetaData WhiskActionMetaData.apply, "namespace", "name", + "_id", "exec", "parameters", "limits", @@ -616,14 +709,19 @@ object WhiskActionMetaData * If it's the actual package, use its name directly as the package path name. * While traversing the package bindings, merge the parameters. */ - def resolveActionAndMergeParameters(entityStore: EntityStore, fullyQualifiedName: FullyQualifiedEntityName)( + def resolveActionAndMergeParameters(entityStore: EntityStore, + fullyQualifiedName: FullyQualifiedEntityName, + version: Option[SemVer] = None)( implicit ec: ExecutionContext, transid: TransactionId): Future[WhiskActionMetaData] = { // first check that there is a package to be resolved val entityPath = fullyQualifiedName.path if (entityPath.defaultPackage) { // this is the default package, nothing to resolve - WhiskActionMetaData.get(entityStore, fullyQualifiedName.toDocId) + WhiskActionVersionList.get(fullyQualifiedName, entityStore).flatMap { result => + val docId = result.matchedDocId(version).getOrElse(fullyQualifiedName.toDocId) + WhiskActionMetaData.get(entityStore, docId) + } } else { // there is a package to be resolved val pkgDocid = fullyQualifiedName.path.toDocId @@ -632,8 +730,12 @@ object WhiskActionMetaData wp flatMap { resolvedPkg => // fully resolved name for the action val fqnAction = resolvedPkg.fullyQualifiedName(withVersion = false).add(actionName) + val action = WhiskActionVersionList.get(fqnAction, entityStore).flatMap { result => + val docId = result.matchedDocId(version).getOrElse(fqnAction.toDocId) + WhiskActionMetaData.get(entityStore, docId) + } // get the whisk action associate with it and inherit the parameters from the package/binding - WhiskActionMetaData.get(entityStore, fqnAction.toDocId) map { + action map { _.inherit( resolvedPkg.parameters, if (fullyQualifiedName.path.equals(resolvedPkg.fullPath)) None diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala index 0fd81d38ef0..da867214918 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala @@ -58,7 +58,7 @@ abstract class WhiskEntity protected[entity] (en: EntityName, val entityType: St FullyQualifiedEntityName(namespace, en, if (withVersion) Some(version) else None) /** The primary key for the entity in the datastore */ - override final def docid = fullyQualifiedName(false).toDocId + override def docid = fullyQualifiedName(false).toDocId /** * Returns a JSON object with the fields specific to this abstract class. diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index a6e4554b071..847f4922c69 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -19,6 +19,7 @@ package org.apache.openwhisk.core.controller import scala.concurrent.Future import scala.concurrent.duration._ +import scala.language.postfixOps import scala.util.{Failure, Success, Try} import org.apache.kafka.common.errors.RecordTooLargeException import akka.actor.ActorSystem @@ -33,7 +34,14 @@ import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.{FeatureFlags, WhiskConfig} import org.apache.openwhisk.core.controller.RestApiCommons.{ListLimit, ListSkip} import org.apache.openwhisk.core.controller.actions.PostActionActivation -import org.apache.openwhisk.core.database.{ActivationStore, CacheChangeNotification, NoDocumentException} +import org.apache.openwhisk.core.database.{ + ActivationStore, + ArtifactStoreException, + CacheChangeNotification, + DocumentConflictException, + DocumentTypeMismatchException, + NoDocumentException +} import org.apache.openwhisk.core.entitlement._ import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.core.entity.types.EntityStore @@ -216,11 +224,20 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case _ => entitlementProvider.check(user, content.exec) } + val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => + result.matchedDocId(None).getOrElse(entityName.toDocId) + } + onComplete(checkAdditionalPrivileges) { case Success(_) => - putEntity(WhiskAction, entityStore, entityName.toDocId, overwrite, update(user, request) _, () => { - make(user, entityName, request) - }) + onComplete(latestDocId) { + case Success(id) => + putEntity(WhiskAction, entityStore, id, true, update(user, request) _, () => { + make(user, entityName, request) + }) + case Failure(f) => + terminate(InternalServerError) + } case Failure(f) => super.handleEntitlementFailure(f) } @@ -244,10 +261,11 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with parameter( 'blocking ? false, 'result ? false, + 'version.as[SemVer] ?, 'timeout.as[FiniteDuration] ? controllerActivationConfig.maxWaitForBlockingActivation) { - (blocking, result, waitOverride) => + (blocking, result, version, waitOverride) => entity(as[Option[JsObject]]) { payload => - getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName), Some { + getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName, version), Some { act: WhiskActionMetaData => // resolve the action --- special case for sequences that may contain components with '_' as default package val action = act.resolve(user.namespace) @@ -333,12 +351,61 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 500 Internal Server Error */ override def remove(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { - deleteEntity(WhiskAction, entityStore, entityName.toDocId, (a: WhiskAction) => Future.successful({})) + parameter('version.as[SemVer] ?) { version => + onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + case Success(results) => + WhiskActionVersionList.deleteCache(entityName) // invalidate version list cache at all cases + version match { + case Some(_) => + val docId = results.matchedDocId(version).getOrElse(entityName.toDocId) + deleteEntity(WhiskAction, entityStore, docId, (a: WhiskAction) => Future.successful({})) + case None => + val fs = + if (results.versions.isEmpty) + Seq(WhiskAction.get(entityStore, entityName.toDocId) flatMap { entity => + WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) + }) + else + results.versions.values + .map { id => + WhiskAction.get(entityStore, DocId(id)) flatMap { entity => + WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) + } + } + onComplete(Future.sequence(fs)) { + case Success(entities) => + complete(OK, entities.last) + case Failure(t: NoDocumentException) => + logging.debug(this, s"[DEL] entity does not exist") + terminate(NotFound) + case Failure(t: DocumentConflictException) => + logging.debug(this, s"[DEL] entity conflict: ${t.getMessage}") + terminate(Conflict, conflictMessage) + case Failure(RejectRequest(code, message)) => + logging.debug(this, s"[DEL] entity rejected with code $code: $message") + terminate(code, message) + case Failure(t: DocumentTypeMismatchException) => + logging.debug(this, s"[DEL] entity conformance check failed: ${t.getMessage}") + terminate(Conflict, conformanceMessage) + case Failure(t: ArtifactStoreException) => + logging.error(this, s"[DEL] entity unreadable") + terminate(InternalServerError, t.getMessage) + case Failure(t: Throwable) => + logging.error(this, s"[DEL] entity failed: ${t.getMessage}") + terminate(InternalServerError) + } + } + case Failure(t) => + terminate(InternalServerError, t.getMessage) + } + } } /** Checks for package binding case. we don't want to allow get for a package binding in shared package */ - private def fetchEntity(entityName: FullyQualifiedEntityName, env: Option[Parameters], code: Boolean)( - implicit transid: TransactionId) = { + private def fetchEntity(entityName: FullyQualifiedEntityName, + env: Option[Parameters], + code: Boolean, + version: Option[SemVer])(implicit transid: TransactionId) = { val resolvedPkg: Future[Either[String, FullyQualifiedEntityName]] = if (entityName.path.defaultPackage) { Future.successful(Right(entityName)) } else { @@ -357,7 +424,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case Left(f) => terminate(Forbidden, f) case Right(_) => if (code) { - getEntity(WhiskAction.resolveActionAndMergeParameters(entityStore, entityName), Some { + getEntity(WhiskAction.resolveActionAndMergeParameters(entityStore, entityName, version), Some { action: WhiskAction => val mergedAction = env map { action inherit _ @@ -365,7 +432,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with complete(OK, mergedAction) }) } else { - getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName), Some { + getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName, version), Some { action: WhiskActionMetaData => val mergedAction = env map { action inherit _ @@ -390,13 +457,13 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with */ override def fetch(user: Identity, entityName: FullyQualifiedEntityName, env: Option[Parameters])( implicit transid: TransactionId) = { - parameter('code ? true) { code => + parameter('code ? true, 'version.as[SemVer] ?) { (code, version) => //check if execute only is enabled, and if there is a discrepancy between the current user's namespace //and that of the entity we are trying to fetch if (executeOnly && user.namespace.name != entityName.namespace) { terminate(Forbidden, forbiddenGetAction(entityName.path.asString)) } else { - fetchEntity(entityName, env, code) + fetchEntity(entityName, env, code, version) } } } @@ -593,7 +660,6 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with content.version getOrElse action.version.upPatch, content.publish getOrElse action.publish, WhiskActionsApi.amendAnnotations(newAnnotations, exec, create = false)) - .revision[WhiskAction](action.docinfo.rev) } /** @@ -693,12 +759,15 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with } else { // check whether component is a sequence or an atomic action // if the component does not exist, the future will fail with appropriate error - WhiskAction.get(entityStore, resolvedComponent.toDocId) flatMap { wskComponent => - wskComponent.exec match { - case SequenceExec(seqComponents) => - // sequence action, count the number of atomic actions in this sequence - countAtomicActionsAndCheckCycle(origSequence, seqComponents) - case _ => Future successful 1 // atomic action count is one + WhiskActionVersionList.get(resolvedComponent, entityStore) flatMap { versions => + val docId = versions.matchedDocId(resolvedComponent.version).getOrElse(resolvedComponent.toDocId) + WhiskAction.get(entityStore, docId) flatMap { wskComponent => + wskComponent.exec match { + case SequenceExec(seqComponents) => + // sequence action, count the number of atomic actions in this sequence + countAtomicActionsAndCheckCycle(origSequence, seqComponents) + case _ => Future successful 1 // atomic action count is one + } } } } @@ -736,6 +805,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with /** Custom unmarshaller for query parameters "skip" for "list" operations. */ private implicit val stringToListSkip: Unmarshaller[String, ListSkip] = RestApiCommons.stringToListSkip(collection) + private implicit val stringToSemVer: Unmarshaller[String, SemVer] = Unmarshaller.strict[String, SemVer] { value => + SemVer(value) + } + } private case class TooManyActionsInSequence() extends IllegalArgumentException diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala index ef2dc51104e..fa4d7bdfc64 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala @@ -395,7 +395,7 @@ trait WriteOps extends Directives { onComplete(factory.get(datastore, docid) flatMap { entity => confirm(entity) flatMap { case _ => - factory.del(datastore, entity.docinfo) map { _ => + factory.del(datastore, docid.asDocInfo(entity.rev)) map { _ => entity } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala index 62b7cacda1a..ab819663404 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala @@ -83,33 +83,42 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { parameter('overwrite ? false) { overwrite => entity(as[WhiskRulePut]) { content => val request = content.resolve(entityName.namespace) + // To avoid using same entity name with action + val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => + result.matchedDocId(None).getOrElse(entityName.toDocId) + } onComplete(entitlementProvider.check(user, Privilege.READ, referencedEntities(request))) { case Success(_) => - putEntity( - WhiskRule, - entityStore, - entityName.toDocId, - overwrite, - update(request) _, - () => { - create(request, entityName) - }, - postProcess = Some { rule: WhiskRule => - if (overwrite == true) { - val getRuleWithStatus = getTrigger(rule.trigger) map { trigger => - getStatus(trigger, FullyQualifiedEntityName(rule.namespace, rule.name)) - } map { status => - rule.withStatus(status) - } - - onComplete(getRuleWithStatus) { - case Success(r) => completeAsRuleResponse(rule, r.status) - case Failure(t) => terminate(InternalServerError) - } - } else { - completeAsRuleResponse(rule, Status.ACTIVE) - } - }) + onComplete(latestDocId) { + case Success(docId) => + putEntity( + WhiskRule, + entityStore, + docId, + overwrite, + update(request) _, + () => { + create(request, entityName) + }, + postProcess = Some { rule: WhiskRule => + if (overwrite == true) { + val getRuleWithStatus = getTrigger(rule.trigger) map { trigger => + getStatus(trigger, FullyQualifiedEntityName(rule.namespace, rule.name)) + } map { status => + rule.withStatus(status) + } + + onComplete(getRuleWithStatus) { + case Success(r) => completeAsRuleResponse(rule, r.status) + case Failure(t) => terminate(InternalServerError) + } + } else { + completeAsRuleResponse(rule, Status.ACTIVE) + } + }) + case Failure(f) => + terminate(InternalServerError) + } case Failure(f) => handleEntitlementFailure(f) } @@ -408,7 +417,16 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { } actionExists <- WhiskAction.resolveAction(entityStore, action) flatMap { resolvedName => - WhiskActionMetaData.get(entityStore, resolvedName.toDocId) + WhiskActionVersionList.get(resolvedName, entityStore).flatMap { versions => + versions + .matchedDocId(resolvedName.version) + .map { docId => + WhiskActionMetaData.get(entityStore, docId) + } + .getOrElse { + WhiskActionMetaData.get(entityStore, resolvedName.toDocId) + } + } } recoverWith { case _: NoDocumentException => Future.failed { diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala index 86b793e477c..2c5429d7d07 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala @@ -21,7 +21,7 @@ import java.time.{Clock, Instant} import scala.collection.immutable.Map import scala.concurrent.Future -import scala.util.Try +import scala.util.{Failure, Success, Try} import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ @@ -46,6 +46,7 @@ import org.apache.openwhisk.core.entitlement.Collection import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.core.entity.types.EntityStore import org.apache.openwhisk.http.ErrorResponse +import org.apache.openwhisk.http.ErrorResponse.terminate import org.apache.openwhisk.http.Messages import org.apache.openwhisk.core.database.UserContext @@ -100,6 +101,9 @@ trait WhiskTriggersApi extends WhiskCollectionAPI { import RestApiCommons.emptyEntityToJsObject + /** JSON response formatter. */ + import RestApiCommons.jsonDefaultResponsePrinter + /** * Creates or updates trigger if it already exists. The PUT content is deserialized into a WhiskTriggerPut * which is a subset of WhiskTrigger (it eschews the namespace and entity name since the former is derived @@ -116,11 +120,21 @@ trait WhiskTriggersApi extends WhiskCollectionAPI { override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { parameter('overwrite ? false) { overwrite => entity(as[WhiskTriggerPut]) { content => - putEntity(WhiskTrigger, entityStore, entityName.toDocId, overwrite, update(content) _, () => { - create(content, entityName) - }, postProcess = Some { trigger => - completeAsTriggerResponse(trigger) - }) + // To avoid using same entity name with action + val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => + result.matchedDocId(None).getOrElse(entityName.toDocId) + } + + onComplete(latestDocId) { + case Success(docId) => + putEntity(WhiskTrigger, entityStore, docId, overwrite, update(content) _, () => { + create(content, entityName) + }, postProcess = Some { trigger => + completeAsTriggerResponse(trigger) + }) + case Failure(f) => + terminate(InternalServerError) + } } } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala index 621a10e2f19..c95e3af9125 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala @@ -175,6 +175,7 @@ protected[actions] trait PrimitiveActions { action.rev, user, activationId, // activation id created here + action.docid, activeAckTopicIndex, waitForResponse.isDefined, args, diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala index 4f4c8ad161c..6cc1dd2b860 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala @@ -416,6 +416,7 @@ class InvokerActor(invokerInstance: InvokerInstanceId, controllerInstance: Contr user = InvokerPool.healthActionIdentity, // Create a new Activation ID for this activation activationId = new ActivationIdGenerator {}.make(), + actionId = action.docid, rootControllerIndex = controllerInstance, blocking = false, content = None, diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala index 7ddff8d2b21..c1d897e8115 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala @@ -169,18 +169,17 @@ class InvokerReactive( def handleActivationMessage(msg: ActivationMessage)(implicit transid: TransactionId): Future[Unit] = { val namespace = msg.action.path val name = msg.action.name - val actionid = FullyQualifiedEntityName(namespace, name).toDocId.asDocInfo(msg.revision) val subject = msg.user.subject - logging.debug(this, s"${actionid.id} $subject ${msg.activationId}") + logging.debug(this, s"${msg.actionId} $subject ${msg.activationId}") // caching is enabled since actions have revision id and an updated // action will not hit in the cache due to change in the revision id; // if the doc revision is missing, then bypass cache - if (actionid.rev == DocRevision.empty) logging.warn(this, s"revision was not provided for ${actionid.id}") + if (msg.revision == DocRevision.empty) logging.warn(this, s"revision was not provided for ${msg.actionId}") WhiskAction - .get(entityStore, actionid.id, actionid.rev, fromCache = actionid.rev != DocRevision.empty) + .get(entityStore, msg.actionId, msg.revision, fromCache = msg.revision != DocRevision.empty) .flatMap(action => { action.toExecutableWhiskAction match { case Some(executable) => diff --git a/tests/src/test/scala/common/WskCliOperations.scala b/tests/src/test/scala/common/WskCliOperations.scala index 925e5d44aac..45e16a38cb1 100644 --- a/tests/src/test/scala/common/WskCliOperations.scala +++ b/tests/src/test/scala/common/WskCliOperations.scala @@ -286,6 +286,7 @@ class CliActionOperations(override val wsk: RunCliCmd) parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult = { val params = Seq(noun, "invoke", "--auth", wp.authKey, fqn(name)) ++ { parameters flatMap { p => diff --git a/tests/src/test/scala/common/WskOperations.scala b/tests/src/test/scala/common/WskOperations.scala index 111f3055bb8..7782745e670 100644 --- a/tests/src/test/scala/common/WskOperations.scala +++ b/tests/src/test/scala/common/WskOperations.scala @@ -252,6 +252,7 @@ trait ActionOperations extends DeleteFromCollectionOperations with ListOrGetFrom parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult } diff --git a/tests/src/test/scala/common/rest/WskRestOperations.scala b/tests/src/test/scala/common/rest/WskRestOperations.scala index 2c652d9c2fd..35e91463c6e 100644 --- a/tests/src/test/scala/common/rest/WskRestOperations.scala +++ b/tests/src/test/scala/common/rest/WskRestOperations.scala @@ -384,8 +384,9 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, expectedExitCode: Int = Accepted.intValue)(implicit wp: WskProps): RestResult = { - super.invokeAction(name, parameters, parameterFile, blocking, result, expectedExitCode = expectedExitCode) + super.invokeAction(name, parameters, parameterFile, blocking, result, version, expectedExitCode = expectedExitCode) } private def readCodeFromFile(artifact: Option[String]): Option[String] = { @@ -1333,6 +1334,7 @@ trait RunRestCmd extends Matchers with ScalaFutures with SwaggerValidator { parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, web: Boolean = false, expectedExitCode: Int = Accepted.intValue)(implicit wp: WskProps): RestResult = { val (ns, actName) = getNamespaceEntityName(name) @@ -1341,7 +1343,9 @@ trait RunRestCmd extends Matchers with ScalaFutures with SwaggerValidator { if (web) Path(s"$basePath/web/$systemNamespace/$actName.http") else Path(s"$basePath/namespaces/$ns/actions/$actName") - val paramMap = Map("blocking" -> blocking.toString, "result" -> result.toString) + val paramMap = Map("blocking" -> blocking.toString, "result" -> result.toString) ++ version + .map(value => Map("version" -> value)) + .getOrElse(Map.empty) val input = parameterFile map { pf => Some(FileUtils.readFileToString(new File(pf), StandardCharsets.UTF_8)) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala index a5660f7df81..7c40b22eb11 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala @@ -49,7 +49,11 @@ class DockerToActivationLogStoreTests extends FlatSpec with Matchers with WskAct val user = Identity(Subject(), Namespace(EntityName("testSpace"), uuid), BasicAuthenticationAuthKey(uuid, Secret())) val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(user.namespace.name.toPath, EntityName("actionName"), exec) + val action = ExecutableWhiskAction( + user.namespace.name.toPath, + EntityName("actionName"), + DocId("testSpace/actionName@0.0.1"), + exec) val successfulActivation = WhiskActivation( user.namespace.name.toPath, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala index eff4b926ded..ed10549b7cc 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala @@ -84,6 +84,7 @@ class ContainerPoolTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret())), ActivationId.generate(), + DocId("asd"), ControllerInstanceId("0"), blocking = false, content = None, @@ -94,11 +95,16 @@ class ContainerPoolTests val invocationNamespace = EntityName("invocationSpace") val differentInvocationNamespace = EntityName("invocationSpace2") - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec) val concurrencyEnabled = Option(WhiskProperties.getProperty("whisk.action.concurrency")).exists(_.toBoolean) val concurrentAction = ExecutableWhiskAction( EntityPath("actionSpace"), EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), exec, limits = ActionLimits(concurrency = ConcurrencyLimit(if (concurrencyEnabled) 3 else 1))) val differentAction = action.copy(name = EntityName("actionName2")) @@ -909,6 +915,7 @@ class ContainerPoolTests val action = ExecutableWhiskAction( EntityPath("actionSpace"), EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), exec, limits = ActionLimits(memory = MemoryLimit(memoryLimit))) val run = createRunMessage(action, invocationNamespace) @@ -999,7 +1006,12 @@ class ContainerPoolObjectTests extends FlatSpec with Matchers with MockFactory { /** Helper to create a new action from String representations */ def createAction(namespace: String = "actionNS", name: String = "actionName", limits: ActionLimits = ActionLimits()) = - ExecutableWhiskAction(EntityPath(namespace), EntityName(name), actionExec, limits = limits) + ExecutableWhiskAction( + EntityPath(namespace), + EntityName(name), + DocId(s"$namespace/$name@0.0.1"), + actionExec, + limits = limits) /** Helper to create WarmedData with sensible defaults */ def warmedData(action: ExecutableWhiskAction = createAction(), diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala index 87aeaf19e7e..92ae72f76a2 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala @@ -84,13 +84,18 @@ class ContainerProxyTests val memoryLimit = 256.MB val invocationNamespace = EntityName("invocationSpace") - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec) val concurrencyEnabled = Option(WhiskProperties.getProperty("whisk.action.concurrency", "false")).exists(_.toBoolean) val testConcurrencyLimit = if (concurrencyEnabled) ConcurrencyLimit(2) else ConcurrencyLimit(1) val concurrentAction = ExecutableWhiskAction( EntityPath("actionSpace"), EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), exec, limits = ActionLimits(concurrency = testConcurrencyLimit)) @@ -122,6 +127,7 @@ class ContainerProxyTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret())), ActivationId.generate(), + DocId("asd"), ControllerInstanceId("0"), blocking = false, content = Some(activationArguments), @@ -2009,7 +2015,12 @@ class ContainerProxyTests val keyFalsyAnnotation = Parameters(Annotations.ProvideApiKeyAnnotationName, JsFalse) val actionWithFalsyKeyAnnotation = - ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec, annotations = keyFalsyAnnotation) + ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec, + annotations = keyFalsyAnnotation) machine ! Run(actionWithFalsyKeyAnnotation, message) expectMsg(Transition(machine, Started, Running)) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index cf38dd8b322..48b7d7696ed 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -202,6 +202,45 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "get action by name and version" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + WhiskActionVersionList.deleteCache(action2.fullyQualifiedName(false)) + + // get latest version if version is not specified + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + Get(s"$collectionPath/${action.name}?version=0.0.2") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + Get(s"$collectionPath/${action.name}?version=0.0.3") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + } + it should "get action with updated field" in { implicit val tid = transid() @@ -337,6 +376,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val expectedWhiskActionMetaData = WhiskActionMetaData( action.namespace, action.name, + action.docid, execMetaData, action.parameters, action.limits, @@ -435,6 +475,52 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "delete action by name and version" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + val action3 = action2.copy(version = action2.version.upPatch) + put(entityStore, action3) + + // delete action@0.0.1, action@0.0.2 and action@0.0.3 should not be deleted + Delete(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + Get(s"/$namespace/${collection.path}/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + + Get(s"/$namespace/${collection.path}/${action.name}?version=0.0.2") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + Get(s"/$namespace/${collection.path}/${action.name}?version=0.0.3") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + // it should delete all actions if version is not specified + Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + } + it should "report NotFound for delete non existent action" in { implicit val tid = transid() Delete(s"$collectionPath/xyz") ~> Route.seal(routes(creds)) ~> check { @@ -639,40 +725,27 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { put(entityStore, action, false) // install the action into the database directly var content = JsObject("exec" -> JsObject("code" -> "".toJson, "kind" -> action.exec.kind.toJson)) + val action2 = action.copy( + version = action.version.upPatch, + annotations = action.annotations ++ Parameters(WhiskAction.execFieldName, action.exec.kind)) Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[WhiskAction] - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - action.exec, - action.parameters, - action.limits, - action.version.upPatch, - action.publish, - action.annotations ++ Parameters(WhiskAction.execFieldName, action.exec.kind))) + checkWhiskEntityResponse(response, action2) } content = """{"annotations":[{"key":"a","value":"B"}]}""".parseJson.asJsObject + val action3 = + action2.copy(annotations = action2.annotations ++ Parameters("a", "B"), version = action2.version.upPatch) Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) + deleteAction(action3.docid) status should be(OK) val response = responseAs[WhiskAction] - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - action.exec, - action.parameters, - action.limits, - action.version.upPatch.upPatch, - action.publish, - action.annotations ++ Parameters("a", "B") ++ Parameters(WhiskAction.execFieldName, action.exec.kind))) + checkWhiskEntityResponse(response, action3) } } @@ -690,9 +763,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val content = WhiskActionPut(Some(jsDefault(""))) put(entityStore, action, false) + val action2 = action.copy(version = action.version.upPatch) + // create an action sequence Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] response.exec.kind should be(NODEJS) @@ -709,9 +785,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val content = WhiskActionPut(Some(jsDefault("")), parameters = Some(Parameters("a", "A"))) put(entityStore, action, false) + val action2 = action.copy(version = action.version.upPatch) + // create an action sequence Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] response.exec.kind should be(NODEJS) @@ -847,7 +926,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { stream.toString should include(s"serving from cache: ${CacheKey(action)}") stream.reset() - // update should invalidate cache + // update should create new cache for new version Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] @@ -864,11 +943,10 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.annotations ++ systemAnnotations(kind))) } stream.toString should include(s"entity exists, will try to update '$action'") - stream.toString should include(s"invalidating ${CacheKey(action)}") - stream.toString should include(s"caching ${CacheKey(action)}") + stream.toString should include(s"caching ${CacheKey(action.copy(version = action.version.upPatch))}") stream.reset() - // delete should invalidate cache + // delete should invalidate cache for all versions Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] @@ -885,6 +963,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.annotations ++ systemAnnotations(kind))) } stream.toString should include(s"invalidating ${CacheKey(action)}") + stream.toString should include(s"invalidating ${CacheKey(action.copy(version = action.version.upPatch))}") stream.reset() } } @@ -1103,6 +1182,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { // second request should not fetch from cache Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check { + deleteAction(action.docid) status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1163,6 +1243,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { (0 until 5).map { i => Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check { + if (i == 4) + deleteAction(action.docid) status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse(response, expectedAction) @@ -1319,16 +1401,6 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } - it should "reject put with conflict for pre-existing action" in { - implicit val tid = transid() - val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) - val content = WhiskActionPut(Some(action.exec)) - put(entityStore, action) - Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { - status should be(Conflict) - } - } - it should "update action with a put" in { implicit val tid = transid() val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"), ActionLimits()) @@ -1342,26 +1414,25 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(LogLimit(LogLimit.MAX_LOGSIZE)), Some(ConcurrencyLimit(ConcurrencyLimit.MAX_CONCURRENT))))) put(entityStore, action) - Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { + + val action2 = action.copy( + exec = content.exec.get, + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS, create = false), + limits = ActionLimits( + content.limits.get.timeout.get, + content.limits.get.memory.get, + content.limits.get.logs.get, + content.limits.get.concurrency.get)) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] response.updated should not be action.updated - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - content.exec.get, - content.parameters.get, - ActionLimits( - content.limits.get.timeout.get, - content.limits.get.memory.get, - content.limits.get.logs.get, - content.limits.get.concurrency.get), - version = action.version.upPatch, - annotations = action.annotations ++ systemAnnotations(NODEJS, create = false))) + checkWhiskEntityResponse(response, action2) } } @@ -1370,19 +1441,53 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) put(entityStore, action) - Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { + val action2 = action.copy( + version = action.version.upPatch, + parameters = content.parameters.get, + annotations = action.annotations ++ systemAnnotations(NODEJS, false)) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - action.exec, - content.parameters.get, - version = action.version.upPatch, - annotations = action.annotations ++ systemAnnotations(NODEJS, false))) + checkWhiskEntityResponse(response, action2) + } + } + + it should "create a new action in database instead of replacing old action" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) + put(entityStore, action) + val action2 = action.copy( + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS, false)) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action2) + } + + // the first version should not be replaced + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // it should update based on latest version, action2 here + val content2 = WhiskActionPut(annotations = Some(Parameters("x", "X"))) + val action3 = + action2.copy(annotations = action2.annotations ++ content2.annotations, version = action2.version.upPatch) + Put(s"$collectionPath/${action.name}", content2) ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + deleteAction(action2.docid) + deleteAction(action3.docid) + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action3) + >>>>>>> Implement action versioning } } @@ -1664,6 +1769,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } put(entityStore, action) + // delete cache for WhiskActionVersionList + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) Put(s"$collectionPath/${action.name}?overwrite=true", JsObject.empty) ~> Route.seal(routes(creds)) ~> check { status shouldBe BadRequest @@ -1689,9 +1796,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } put(entityStore, action) + // delete cache for WhiskActionVersionList + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) Put(s"$collectionPath/${action.name}?overwrite=true", okUpdate) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action.copy(version = action.version.upPatch).docid) status shouldBe OK } } finally { diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala index 5d48d42b1ae..492a0fc87ef 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala @@ -335,15 +335,15 @@ class EntitlementProviderTests extends ControllerTestCommon with ScalaFutures { (REJECT, guestUser, Right(false))) // this forces a doc mismatch error - val action = WhiskAction(someUser.namespace.name.toPath, MakeName.next(), jsDefault("")) - put(entityStore, action) + val trigger = WhiskTrigger(someUser.namespace.name.toPath, MakeName.next()) + put(entityStore, trigger) paths foreach { case (priv, who, expected) => val check = new PackageCollection(entityStore).implicitRights( who, Set(who.namespace.name.asString), priv, - Resource(someUser.namespace.name.toPath, PACKAGES, Some(action.name.asString))) + Resource(someUser.namespace.name.toPath, PACKAGES, Some(trigger.name.asString))) Await.ready(check, requestTimeout).eitherValue.get shouldBe expected } } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala index 578e7ff2651..ae725ed0f7b 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala @@ -862,11 +862,11 @@ class PackagesApiTests extends ControllerTestCommon with WhiskPackagesApi { it should "reject bind to non-package" in { implicit val tid = transid() - val action = WhiskAction(namespace, aname(), jsDefault("??")) - val reference = WhiskPackage(namespace, aname(), Some(Binding(action.namespace.root, action.name))) + val trigger = WhiskTrigger(namespace, aname()) + val reference = WhiskPackage(namespace, aname(), Some(Binding(trigger.namespace.root, trigger.name))) val content = WhiskPackagePut(reference.binding) - put(entityStore, action) + put(entityStore, trigger) Put(s"$collectionPath/${reference.name}", content) ~> Route.seal(routes(creds)) ~> check { status should be(Conflict) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala index 896c14a73c2..02c80bca4a8 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala @@ -65,12 +65,12 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { end = Instant.now) putSimpleSequenceInDB(seqName, namespace, Vector(compName1, compName2)) - deleteAction(DocId(s"$namespace/$compName2")) + deleteAction(DocId(s"$namespace/$compName2@0.0.1")) loadBalancer.whiskActivationStub = Some((1.milliseconds, Right(comp1Activation))) Post(s"$collectionPath/$seqName?blocking=true") ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/$seqName")) - deleteAction(DocId(s"$namespace/$compName1")) + deleteAction(DocId(s"$namespace/$seqName@0.0.1")) + deleteAction(DocId(s"$namespace/$compName1@0.0.1")) status should be(BadGateway) val response = responseAs[JsObject] response.fields("response") shouldBe ActivationResponse.applicationError(sequenceComponentNotFound).toExtendedJson @@ -196,7 +196,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // update the sequence Put(s"$collectionPath/${seqName.name}?overwrite=true", updatedContent) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/${seqName.name}")) + deleteAction(DocId(s"$namespace/${seqName.name}@0.0.1")) status should be(BadRequest) responseAs[ErrorResponse].error shouldBe Messages.sequenceIsCyclic } @@ -215,7 +215,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // create an action sequence Put(s"$collectionPath/${seqName.name}", content) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/${seqName.name}")) + deleteAction(DocId(s"$namespace/${seqName.name}@0.0.1")) status should be(OK) } } @@ -246,7 +246,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // create an action sequence Put(s"$collectionPath/${seqName.name}", content) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/${seqName.name}")) + deleteAction(DocId(s"$namespace/${seqName.name}@0.0.1")) status should be(OK) val response = responseAs[String] } @@ -285,7 +285,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // update the sequence Put(s"$collectionPath/$pkg/${seqName.name}?overwrite=true", updatedContent) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/$pkg/${seqName.name}")) + deleteAction(DocId(s"$namespace/$pkg/${seqName.name}@0.0.1")) status should be(BadRequest) responseAs[ErrorResponse].error shouldBe Messages.sequenceIsCyclic } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala index ee25ece3a63..13056eb57ae 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala @@ -561,24 +561,27 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with ExecHelpers with Mat } it should "exclude undefined code in whisk action initializer" in { - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1")).containerInitializer() shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1")) + .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "main" -> "main".toJson) } - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1", "xyz")).containerInitializer() shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1", "xyz")) + .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "main" -> "main".toJson, "code" -> "xyz".toJson) } - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1", "", Some("naim"))) + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1", "", Some("naim"))) .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "main" -> "naim".toJson) } } it should "allow of main override in action initializer" in { - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), jsDefault("")).containerInitializer() shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), jsDefault("")) + .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "code" -> JsString.empty, "main" -> "main".toJson) } - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), jsDefault("", Some("bar"))) + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), jsDefault("", Some("bar"))) .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "code" -> JsString.empty, "main" -> "bar".toJson) } @@ -593,7 +596,8 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with ExecHelpers with Mat "E" -> JsArray(JsString("a")), "F" -> JsObject("a" -> JsFalse)) - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1")).containerInitializer(env) shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1")) + .containerInitializer(env) shouldBe { JsObject( "name" -> "b".toJson, "binary" -> false.toJson, @@ -620,7 +624,11 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with ExecHelpers with Mat it should "compare as equal two executable actions even if their revision does not match" in { val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val actionA = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val actionA = ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec) val actionB = actionA.copy() val actionC = actionA.copy() actionC.revision(DocRevision("2")) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala index a4a6145e5f5..39c1ae301cd 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala @@ -190,6 +190,7 @@ class InvokerSupervisionTests Namespace(EntityName("unhealthyInvokerCheck"), uuid), BasicAuthenticationAuthKey(uuid, Secret())), activationId = new ActivationIdGenerator {}.make(), + DocId("asd"), rootControllerIndex = ControllerInstanceId("0"), blocking = false, content = None, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala index dbadbce6e9b..934cd23f6e8 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala @@ -37,8 +37,10 @@ import org.scalatest.Matchers import scala.concurrent.Await import scala.concurrent.Future import scala.concurrent.duration._ -import org.apache.openwhisk.common.{InvokerHealth, Logging, NestedSemaphore, TransactionId} -import org.apache.openwhisk.core.entity.FullyQualifiedEntityName +import org.apache.openwhisk.common.Logging +import org.apache.openwhisk.common.NestedSemaphore +import org.apache.openwhisk.core.entity._ +import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.WhiskConfig import org.apache.openwhisk.core.connector.ActivationMessage import org.apache.openwhisk.core.connector.CompletionMessage @@ -46,22 +48,7 @@ import org.apache.openwhisk.core.connector.Message import org.apache.openwhisk.core.connector.MessageConsumer import org.apache.openwhisk.core.connector.MessageProducer import org.apache.openwhisk.core.connector.MessagingProvider -import org.apache.openwhisk.core.entity.ActivationId -import org.apache.openwhisk.core.entity.BasicAuthenticationAuthKey -import org.apache.openwhisk.core.entity.ControllerInstanceId -import org.apache.openwhisk.core.entity.EntityName -import org.apache.openwhisk.core.entity.EntityPath -import org.apache.openwhisk.core.entity.ExecManifest -import org.apache.openwhisk.core.entity.Identity -import org.apache.openwhisk.core.entity.InvokerInstanceId -import org.apache.openwhisk.core.entity.MemoryLimit -import org.apache.openwhisk.core.entity.Namespace -import org.apache.openwhisk.core.entity.Secret -import org.apache.openwhisk.core.entity.Subject -import org.apache.openwhisk.core.entity.UUID -import org.apache.openwhisk.core.entity.WhiskActionMetaData import org.apache.openwhisk.core.entity.test.ExecHelpers -import org.apache.openwhisk.core.entity.ByteSize import org.apache.openwhisk.core.entity.size._ import org.apache.openwhisk.core.entity.test.ExecHelpers import org.apache.openwhisk.core.loadBalancer.FeedFactory @@ -419,6 +406,7 @@ class ShardingContainerPoolBalancerTests WhiskActionMetaData( namespace, name, + DocId(s"$namespace/$name@0.0.1"), jsMetaData(Some("jsMain"), false), limits = actionLimits(actionMem, concurrency)) val maxContainers = invokerMem.toMB.toInt / actionMetaData.limits.memory.megabytes @@ -503,6 +491,7 @@ class ShardingContainerPoolBalancerTests actionMetaData.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret())), aid, + DocId("asd"), ControllerInstanceId("0"), blocking = false, content = None, diff --git a/tests/src/test/scala/system/basic/WskActionTests.scala b/tests/src/test/scala/system/basic/WskActionTests.scala index 4b4d60376bf..4d25d798bcc 100644 --- a/tests/src/test/scala/system/basic/WskActionTests.scala +++ b/tests/src/test/scala/system/basic/WskActionTests.scala @@ -20,6 +20,7 @@ package system.basic import java.io.File import java.nio.charset.StandardCharsets +import akka.http.scaladsl.model.StatusCodes.NotFound import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import common._ @@ -51,6 +52,37 @@ class WskActionTests extends TestHelpers with WskTestHelpers with JsHelpers with } } + it should "save multi versions for an action and invoke them" in withAssetCleaner(wskprops) { (wp, assetHelper) => + val name = "hello" + assetHelper.withCleaner(wsk.action, name) { (action, _) => + action.create(name, Some(TestUtils.getTestActionFilename("hello.js"))) + action.create(name, Some(TestUtils.getTestActionFilename("echo.js"))) + } + + // invoke the latest version + var run = wsk.action.invoke(name, Map("payload" -> "world".toJson)) + withActivation(wsk.activation, run) { activation => + activation.response.status shouldBe "success" + activation.response.result shouldBe Some(JsObject("payload" -> "world".toJson)) + activation.logs.get.mkString(" ") shouldBe empty + } + + // invoke the first version + run = wsk.action.invoke(name, Map("payload" -> "world".toJson), version = Some("0.0.1")) + withActivation(wsk.activation, run) { activation => + activation.response.status shouldBe "success" + activation.response.result shouldBe Some(JsObject("payload" -> "hello, world!".toJson)) + activation.logs.get.mkString(" ") should include(s"hello, world!") + } + + // invoke a non-exist version + wsk.action.invoke( + name, + Map("payload" -> "world".toJson), + version = Some("0.0.3"), + expectedExitCode = NotFound.intValue) + } + it should "invoke an action returning a promise" in withAssetCleaner(wskprops) { (wp, assetHelper) => val name = "hello promise" assetHelper.withCleaner(wsk.action, name) { (action, _) => diff --git a/tests/src/test/scala/system/basic/WskRestBasicTests.scala b/tests/src/test/scala/system/basic/WskRestBasicTests.scala index 134808928ca..abb98ce8cba 100644 --- a/tests/src/test/scala/system/basic/WskRestBasicTests.scala +++ b/tests/src/test/scala/system/basic/WskRestBasicTests.scala @@ -282,19 +282,6 @@ class WskRestBasicTests extends TestHelpers with WskTestHelpers with WskActorSys actions.exists(action => RestResult.getField(action, "name") == name) shouldBe true } - it should "reject create of an action that already exists" in withAssetCleaner(wskprops) { - val name = "dupeAction" - val file = Some(TestUtils.getTestActionFilename("echo.js")) - - (wp, assetHelper) => - assetHelper.withCleaner(wsk.action, name) { (action, _) => - action.create(name, file) - } - - val stderr = wsk.action.create(name, file, expectedExitCode = Conflict.intValue).stderr - stderr should include("resource already exists") - } - it should "reject delete of action that does not exist" in { val name = "nonexistentAction" val stderr = wsk.action.delete(name, expectedExitCode = NotFound.intValue).stderr From 8409b85d8a5dcf311af50ec60d4d63d1864cd0d5 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 28 Sep 2020 16:27:03 +0800 Subject: [PATCH 02/29] Fix tests --- .../openwhisk/core/entity/WhiskAction.scala | 6 +--- .../openwhisk/core/controller/Actions.scala | 22 ++++++++++-- .../openwhisk/core/controller/Packages.scala | 35 ++++++++++++------- .../core/entitlement/PackageCollection.scala | 7 ++-- .../test/EntitlementProviderTests.scala | 6 ++-- .../controller/test/PackagesApiTests.scala | 6 ++-- 6 files changed, 54 insertions(+), 28 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index f0fbf6fd02a..1c21c6957d6 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -495,11 +495,7 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ old) } } match { - case Success(f) => - implicit val ec = db.executionContext - implicit val logger = db.logging - WhiskActionVersionList.deleteCache(doc.fullyQualifiedName(false)) - f + case Success(f) => f case Failure(f) => Future.failed(f) } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 847f4922c69..46031672022 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -234,6 +234,9 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case Success(id) => putEntity(WhiskAction, entityStore, id, true, update(user, request) _, () => { make(user, entityName, request) + }, postProcess = Some { action: WhiskAction => + WhiskActionVersionList.deleteCache(entityName) + complete(OK, action) }) case Failure(f) => terminate(InternalServerError) @@ -354,11 +357,18 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with parameter('version.as[SemVer] ?) { version => onComplete(WhiskActionVersionList.get(entityName, entityStore)) { case Success(results) => - WhiskActionVersionList.deleteCache(entityName) // invalidate version list cache at all cases version match { case Some(_) => val docId = results.matchedDocId(version).getOrElse(entityName.toDocId) - deleteEntity(WhiskAction, entityStore, docId, (a: WhiskAction) => Future.successful({})) + deleteEntity( + WhiskAction, + entityStore, + docId, + (a: WhiskAction) => Future.successful({}), + postProcess = Some { action: WhiskAction => + WhiskActionVersionList.deleteCache(entityName) + complete(OK, action) + }) case None => val fs = if (results.versions.isEmpty) @@ -372,7 +382,13 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) } } - onComplete(Future.sequence(fs)) { + val deleteFuture = Future.sequence(fs).andThen { + case _ => + WhiskActionVersionList + .deleteCache(entityName) // invalidate version list cache after all deletion completed + } + + onComplete(deleteFuture) { case Success(entities) => complete(OK, entities.last) case Failure(t: NoDocumentException) => diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala index c524e9570b1..4d613328c4a 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala @@ -85,15 +85,25 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { } val referencedentities = referencedEntities(request) + // To avoid using same entity name with action + val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => + result.matchedDocId(None).getOrElse(entityName.toDocId) + } + onComplete(entitlementProvider.check(user, Privilege.READ, referencedentities)) { case Success(_) => - putEntity( - WhiskPackage, - entityStore, - entityName.toDocId, - overwrite, - update(request) _, - () => create(request, entityName)) + onComplete(latestDocId) { + case Success(docId) => + putEntity( + WhiskPackage, + entityStore, + docId, + overwrite, + update(request) _, + () => create(request, entityName)) + case Failure(f) => + terminate(InternalServerError) + } case Failure(f) => rewriteEntitlementFailure(f) } @@ -149,14 +159,15 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { case Right(list) if list.nonEmpty && force => Future sequence { list.map(action => { - WhiskAction.get( - entityStore, - wp.fullyQualifiedName(false) - .add(action.fullyQualifiedName(false).name) - .toDocId) flatMap { actionWithRevision => + WhiskAction.get(entityStore, action.docid) flatMap { actionWithRevision => WhiskAction.del(entityStore, actionWithRevision.docinfo) } }) + } andThen { + case _ => + list.foreach { action => + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) + } } flatMap { _ => Future.successful({}) } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala index b53dcbc7d37..642e616a274 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala @@ -62,8 +62,11 @@ class PackageCollection(entityStore: EntityStore)(implicit logging: Logging) ext case Privilege.READ => // must determine if this is a public or owned package // or, for a binding, that it references a public or owned package - val docid = FullyQualifiedEntityName(resource.namespace.root.toPath, EntityName(pkgname)).toDocId - checkPackageReadPermission(namespaces, isOwner, docid) + val entityName = FullyQualifiedEntityName(resource.namespace.root.toPath, EntityName(pkgname)) + WhiskActionVersionList.get(entityName, entityStore).flatMap { result => + val docid = result.matchedDocId(None).getOrElse(entityName.toDocId) + checkPackageReadPermission(namespaces, isOwner, docid) + } case _ => Future.successful(isOwner && allowedEntityRights.contains(right)) } } getOrElse { diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala index 492a0fc87ef..5d48d42b1ae 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/EntitlementProviderTests.scala @@ -335,15 +335,15 @@ class EntitlementProviderTests extends ControllerTestCommon with ScalaFutures { (REJECT, guestUser, Right(false))) // this forces a doc mismatch error - val trigger = WhiskTrigger(someUser.namespace.name.toPath, MakeName.next()) - put(entityStore, trigger) + val action = WhiskAction(someUser.namespace.name.toPath, MakeName.next(), jsDefault("")) + put(entityStore, action) paths foreach { case (priv, who, expected) => val check = new PackageCollection(entityStore).implicitRights( who, Set(who.namespace.name.asString), priv, - Resource(someUser.namespace.name.toPath, PACKAGES, Some(trigger.name.asString))) + Resource(someUser.namespace.name.toPath, PACKAGES, Some(action.name.asString))) Await.ready(check, requestTimeout).eitherValue.get shouldBe expected } } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala index ae725ed0f7b..578e7ff2651 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/PackagesApiTests.scala @@ -862,11 +862,11 @@ class PackagesApiTests extends ControllerTestCommon with WhiskPackagesApi { it should "reject bind to non-package" in { implicit val tid = transid() - val trigger = WhiskTrigger(namespace, aname()) - val reference = WhiskPackage(namespace, aname(), Some(Binding(trigger.namespace.root, trigger.name))) + val action = WhiskAction(namespace, aname(), jsDefault("??")) + val reference = WhiskPackage(namespace, aname(), Some(Binding(action.namespace.root, action.name))) val content = WhiskPackagePut(reference.binding) - put(entityStore, trigger) + put(entityStore, action) Put(s"$collectionPath/${reference.name}", content) ~> Route.seal(routes(creds)) ~> check { status should be(Conflict) From 7d72c5337564d172911439b61678471a23a9112d Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Tue, 29 Sep 2020 13:56:29 +0800 Subject: [PATCH 03/29] Make SemVer sortable --- .../apache/openwhisk/core/entity/SemVer.scala | 22 ++++++++++++++++++- .../openwhisk/core/entity/WhiskAction.scala | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala index 7cbf1c802b0..334ae0305b6 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala @@ -21,6 +21,8 @@ import spray.json.deserializationError import spray.json.JsString import spray.json.JsValue import spray.json.RootJsonFormat + +import scala.annotation.tailrec import scala.util.Try /** @@ -34,7 +36,7 @@ import scala.util.Try * * @param (major, minor, patch) for the semantic version */ -protected[core] class SemVer private (private val version: (Int, Int, Int)) extends AnyVal { +protected[core] class SemVer private (private val version: (Int, Int, Int)) extends AnyVal with Ordered[SemVer] { protected[core] def major = version._1 protected[core] def minor = version._2 @@ -46,6 +48,24 @@ protected[core] class SemVer private (private val version: (Int, Int, Int)) exte protected[entity] def toJson = JsString(toString) override def toString = s"$major.$minor.$patch" + + protected[core] def versionList = Seq(major, minor, patch) + + override def compare(that: SemVer): Int = { + compareVersion(that) + } + + @tailrec + private def compareVersion(that: SemVer, depth: Int = 0): Int = { + require(depth >= 0 && depth <= 2, "depth exceed the limit of 0 to 2") + val result = versionList(depth) - that.versionList(depth) + if (result != 0) + result + else if (depth == 2) + 0 + else + compareVersion(that, depth + 1) + } } protected[core] object SemVer { diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 1c21c6957d6..796e641717f 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -370,7 +370,7 @@ case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versi case Some(ver) => versions.get(ver).map(DocId(_)) case None if versions.nonEmpty => - Some(DocId(versions.maxBy(_._1.toString)._2)) + Some(DocId(versions.maxBy(_._1)._2)) case _ => None } From 595df673f6a987814b56d76b704726d7fb67486a Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Fri, 9 Oct 2020 17:00:42 +0800 Subject: [PATCH 04/29] Updates: 1. Only return "published" version action 2. Add max version limit --- ...esign_document_for_entities_db_v2.1.0.json | 2 +- .../scala/src/main/resources/application.conf | 1 + .../apache/openwhisk/core/WhiskConfig.scala | 1 + .../openwhisk/core/connector/Message.scala | 2 +- .../openwhisk/core/entity/WhiskAction.scala | 42 +++++++---- .../openwhisk/core/controller/Actions.scala | 62 ++++++++++------ .../openwhisk/core/controller/Packages.scala | 9 +-- .../openwhisk/core/controller/Rules.scala | 19 ++--- .../openwhisk/core/controller/Triggers.scala | 24 ++++--- .../core/entitlement/PackageCollection.scala | 5 +- .../controller/test/ActionsApiTests.scala | 72 ++++++++++++++++++- 11 files changed, 162 insertions(+), 77 deletions(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index ccac41a016f..ecbdc5d9d4f 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -23,7 +23,7 @@ "reduce": "_count" }, "action-versions": { - "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n id: doc._id,\n version: doc.version,\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var published = true;\n doc.annotations.forEach(function(anno) {\n if(anno[\"key\"] == \"publish\")\n published = anno[\"value\"]\n });\n \n var value = {\n namespace: doc.namespace,\n name: doc.name,\n _id: doc._id,\n version: doc.version,\n publish: published,\n };\n var versions = doc.version.split(\".\")\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } } diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf index 230e16d4308..7e603c7efef 100644 --- a/common/scala/src/main/resources/application.conf +++ b/common/scala/src/main/resources/application.conf @@ -100,6 +100,7 @@ kamon { whisk { shared-packages-execute-only = false + action-maximum-versions = 10 metrics { # Enable/disable Prometheus support. If enabled then metrics would be exposed at `/metrics` endpoint # If Prometheus is enabled then please review `kamon.metric.tick-interval` (set to 1 sec by default above). diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index c7ec2066287..ca90b855db0 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -283,6 +283,7 @@ object ConfigKeys { val whiskConfig = "whisk.config" val sharedPackageExecuteOnly = s"whisk.shared-packages-execute-only" + val actionVersionLimit = "whisk.action-maximum-versions" val swaggerUi = "whisk.swagger-ui" val disableStoreResult = s"$activation.disable-store-result" diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala index 2e28d3d141c..b01e60a04ae 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala @@ -192,7 +192,7 @@ object ActivationMessage extends DefaultJsonProtocol { def parse(msg: String) = Try(serdes.read(msg.parseJson)) private implicit val fqnSerdes = FullyQualifiedEntityName.serdes - implicit val serdes = jsonFormat12(ActivationMessage.apply) + implicit val serdes = jsonFormat13(ActivationMessage.apply) } object CombinedCompletionAndResultMessage extends DefaultJsonProtocol { diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 796e641717f..4eb44f48f6b 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -358,10 +358,10 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, } -case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer) +case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer, publish: Boolean) object WhiskActionVersion { - val serdes = jsonFormat4(WhiskActionVersion.apply) + val serdes = jsonFormat5(WhiskActionVersion.apply) } case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versions: Map[SemVer, String]) { @@ -384,7 +384,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi CacheKey(action.fullPath.asString) } - def get(action: FullyQualifiedEntityName, datastore: EntityStore)( + def get(action: FullyQualifiedEntityName, datastore: EntityStore, fetchAll: Boolean = true)( implicit transId: TransactionId): Future[WhiskActionVersionList] = { implicit val logger: Logging = datastore.logging implicit val ec = datastore.executionContext @@ -409,6 +409,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi } val mappings = values .map(WhiskActionVersion.serdes.read(_)) + .filter(_.publish || fetchAll) .map { actionVersion => (actionVersion.version, actionVersion.id) } @@ -417,6 +418,21 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi }) } + def getMatchedDocId(action: FullyQualifiedEntityName, version: Option[SemVer], datastore: EntityStore)( + implicit transId: TransactionId, + ec: ExecutionContext): Future[Option[DocId]] = { + get(action, datastore, version.nonEmpty).map { res => + version match { + case Some(ver) => + res.versions.get(ver).map(DocId(_)) + case None if res.versions.nonEmpty => + Some(DocId(res.versions.maxBy(_._1)._2)) + case _ => + None + } + } + } + // delete cache def deleteCache(action: FullyQualifiedEntityName)(implicit transId: TransactionId, ec: ExecutionContext, @@ -625,9 +641,8 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ val entityPath = fullyQualifiedName.path if (entityPath.defaultPackage) { // this is the default package, nothing to resolve - WhiskActionVersionList.get(fullyQualifiedName, entityStore).flatMap { result => - val docId = result.matchedDocId(version).getOrElse(fullyQualifiedName.toDocId) - WhiskAction.get(entityStore, docId) + WhiskActionVersionList.getMatchedDocId(fullyQualifiedName, version, entityStore).flatMap { docId => + WhiskAction.get(entityStore, docId.getOrElse(fullyQualifiedName.toDocId)) } } else { // there is a package to be resolved @@ -637,9 +652,8 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ wp flatMap { resolvedPkg => // fully resolved name for the action val fqnAction = resolvedPkg.fullyQualifiedName(withVersion = false).add(actionName) - val action = WhiskActionVersionList.get(fqnAction, entityStore).flatMap { result => - val docId = result.matchedDocId(version).getOrElse(fqnAction.toDocId) - WhiskAction.get(entityStore, docId) + val action = WhiskActionVersionList.getMatchedDocId(fqnAction, version, entityStore).flatMap { docId => + WhiskAction.get(entityStore, docId.getOrElse(fqnAction.toDocId)) } // get the whisk action associate with it and inherit the parameters from the package/binding action map { @@ -714,9 +728,8 @@ object WhiskActionMetaData val entityPath = fullyQualifiedName.path if (entityPath.defaultPackage) { // this is the default package, nothing to resolve - WhiskActionVersionList.get(fullyQualifiedName, entityStore).flatMap { result => - val docId = result.matchedDocId(version).getOrElse(fullyQualifiedName.toDocId) - WhiskActionMetaData.get(entityStore, docId) + WhiskActionVersionList.getMatchedDocId(fullyQualifiedName, version, entityStore).flatMap { docId => + WhiskActionMetaData.get(entityStore, docId.getOrElse(fullyQualifiedName.toDocId)) } } else { // there is a package to be resolved @@ -726,9 +739,8 @@ object WhiskActionMetaData wp flatMap { resolvedPkg => // fully resolved name for the action val fqnAction = resolvedPkg.fullyQualifiedName(withVersion = false).add(actionName) - val action = WhiskActionVersionList.get(fqnAction, entityStore).flatMap { result => - val docId = result.matchedDocId(version).getOrElse(fqnAction.toDocId) - WhiskActionMetaData.get(entityStore, docId) + val action = WhiskActionVersionList.getMatchedDocId(fqnAction, version, entityStore).flatMap { docId => + WhiskActionMetaData.get(entityStore, docId.getOrElse(fqnAction.toDocId)) } // get the whisk action associate with it and inherit the parameters from the package/binding action map { diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 46031672022..e6d97bb1dd8 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -116,6 +116,9 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with protected def executeOnly = loadConfigOrThrow[Boolean](ConfigKeys.sharedPackageExecuteOnly) + private val actionMaxVersionLimit = + loadConfigOrThrow[Int](ConfigKeys.actionVersionLimit) + /** Entity normalizer to JSON object. */ import RestApiCommons.emptyEntityToJsObject @@ -224,20 +227,35 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case _ => entitlementProvider.check(user, content.exec) } - val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => - result.matchedDocId(None).getOrElse(entityName.toDocId) - } - onComplete(checkAdditionalPrivileges) { case Success(_) => - onComplete(latestDocId) { - case Success(id) => - putEntity(WhiskAction, entityStore, id, true, update(user, request) _, () => { - make(user, entityName, request) - }, postProcess = Some { action: WhiskAction => - WhiskActionVersionList.deleteCache(entityName) - complete(OK, action) - }) + onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + case Success(result) => + val id = result.matchedDocId(None).getOrElse(entityName.toDocId) + putEntity( + WhiskAction, + entityStore, + id, + true, + update(user, request) _, + () => { + make(user, entityName, request) + }, + postProcess = Some { action: WhiskAction => + // delete oldest version when created successfully + if (result.versions.size >= actionMaxVersionLimit) { + val id = result.versions.minBy(_._1)._2 + WhiskAction.get(entityStore, DocId(id)) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) + } andThen { + case _ => + WhiskActionVersionList.deleteCache(entityName) + } + } else { + WhiskActionVersionList.deleteCache(entityName) + } + complete(OK, action) + }) case Failure(f) => terminate(InternalServerError) } @@ -379,7 +397,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with results.versions.values .map { id => WhiskAction.get(entityStore, DocId(id)) flatMap { entity => - WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) + WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) } } val deleteFuture = Future.sequence(fs).andThen { @@ -775,16 +793,16 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with } else { // check whether component is a sequence or an atomic action // if the component does not exist, the future will fail with appropriate error - WhiskActionVersionList.get(resolvedComponent, entityStore) flatMap { versions => - val docId = versions.matchedDocId(resolvedComponent.version).getOrElse(resolvedComponent.toDocId) - WhiskAction.get(entityStore, docId) flatMap { wskComponent => - wskComponent.exec match { - case SequenceExec(seqComponents) => - // sequence action, count the number of atomic actions in this sequence - countAtomicActionsAndCheckCycle(origSequence, seqComponents) - case _ => Future successful 1 // atomic action count is one + WhiskActionVersionList.getMatchedDocId(resolvedComponent, resolvedComponent.version, entityStore) flatMap { + docId => + WhiskAction.get(entityStore, docId.getOrElse(resolvedComponent.toDocId)) flatMap { wskComponent => + wskComponent.exec match { + case SequenceExec(seqComponents) => + // sequence action, count the number of atomic actions in this sequence + countAtomicActionsAndCheckCycle(origSequence, seqComponents) + case _ => Future successful 1 // atomic action count is one + } } - } } } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala index 4d613328c4a..68a19512dc1 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala @@ -85,19 +85,14 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { } val referencedentities = referencedEntities(request) - // To avoid using same entity name with action - val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => - result.matchedDocId(None).getOrElse(entityName.toDocId) - } - onComplete(entitlementProvider.check(user, Privilege.READ, referencedentities)) { case Success(_) => - onComplete(latestDocId) { + onComplete(WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore)) { case Success(docId) => putEntity( WhiskPackage, entityStore, - docId, + docId.getOrElse(entityName.toDocId), overwrite, update(request) _, () => create(request, entityName)) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala index ab819663404..8a5654e25dd 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala @@ -83,18 +83,14 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { parameter('overwrite ? false) { overwrite => entity(as[WhiskRulePut]) { content => val request = content.resolve(entityName.namespace) - // To avoid using same entity name with action - val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => - result.matchedDocId(None).getOrElse(entityName.toDocId) - } onComplete(entitlementProvider.check(user, Privilege.READ, referencedEntities(request))) { case Success(_) => - onComplete(latestDocId) { + onComplete(WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore)) { case Success(docId) => putEntity( WhiskRule, entityStore, - docId, + docId.getOrElse(entityName.toDocId), overwrite, update(request) _, () => { @@ -417,15 +413,8 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { } actionExists <- WhiskAction.resolveAction(entityStore, action) flatMap { resolvedName => - WhiskActionVersionList.get(resolvedName, entityStore).flatMap { versions => - versions - .matchedDocId(resolvedName.version) - .map { docId => - WhiskActionMetaData.get(entityStore, docId) - } - .getOrElse { - WhiskActionMetaData.get(entityStore, resolvedName.toDocId) - } + WhiskActionVersionList.getMatchedDocId(resolvedName, resolvedName.version, entityStore).flatMap { docId => + WhiskActionMetaData.get(entityStore, docId.getOrElse(resolvedName.toDocId)) } } recoverWith { case _: NoDocumentException => diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala index 2c5429d7d07..70043870ce7 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala @@ -120,18 +120,20 @@ trait WhiskTriggersApi extends WhiskCollectionAPI { override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { parameter('overwrite ? false) { overwrite => entity(as[WhiskTriggerPut]) { content => - // To avoid using same entity name with action - val latestDocId = WhiskActionVersionList.get(entityName, entityStore).map { result => - result.matchedDocId(None).getOrElse(entityName.toDocId) - } - - onComplete(latestDocId) { + onComplete(WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore)) { case Success(docId) => - putEntity(WhiskTrigger, entityStore, docId, overwrite, update(content) _, () => { - create(content, entityName) - }, postProcess = Some { trigger => - completeAsTriggerResponse(trigger) - }) + putEntity( + WhiskTrigger, + entityStore, + docId.getOrElse(entityName.toDocId), + overwrite, + update(content) _, + () => { + create(content, entityName) + }, + postProcess = Some { trigger => + completeAsTriggerResponse(trigger) + }) case Failure(f) => terminate(InternalServerError) } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala index 642e616a274..f541d912dfb 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala @@ -63,9 +63,8 @@ class PackageCollection(entityStore: EntityStore)(implicit logging: Logging) ext // must determine if this is a public or owned package // or, for a binding, that it references a public or owned package val entityName = FullyQualifiedEntityName(resource.namespace.root.toPath, EntityName(pkgname)) - WhiskActionVersionList.get(entityName, entityStore).flatMap { result => - val docid = result.matchedDocId(None).getOrElse(entityName.toDocId) - checkPackageReadPermission(namespaces, isOwner, docid) + WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore).flatMap { docId => + checkPackageReadPermission(namespaces, isOwner, docId.getOrElse(entityName.toDocId)) } case _ => Future.successful(isOwner && allowedEntityRights.contains(right)) } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index 48b7d7696ed..0781ec0408c 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -29,6 +29,9 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport.sprayJsonUnmars import akka.http.scaladsl.server.Route import spray.json._ import spray.json.DefaultJsonProtocol._ +import org.apache.commons.lang3.StringUtils +import org.apache.openwhisk.core.ConfigKeys +import org.apache.openwhisk.core.connector.ActivationMessage import org.apache.openwhisk.core.controller.WhiskActionsApi import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.core.entity.size._ @@ -37,11 +40,12 @@ import org.apache.openwhisk.http.ErrorResponse import org.apache.openwhisk.http.Messages import org.apache.openwhisk.core.database.UserContext import akka.http.scaladsl.model.headers.RawHeader -import org.apache.commons.lang3.StringUtils -import org.apache.openwhisk.core.connector.ActivationMessage import org.apache.openwhisk.core.entity.Attachments.Inline import org.apache.openwhisk.core.entity.test.ExecHelpers import org.scalatest.{FlatSpec, Matchers} +import pureconfig.loadConfigOrThrow + +import scala.language.postfixOps /** * Tests Actions API. @@ -241,6 +245,40 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "not get unpublished version" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + val action2 = action.copy(version = action.version.upPatch, annotations = Parameters("publish", JsFalse)) + put(entityStore, action2) + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) + + // the action2 should not be returned + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + val action3 = action.copy(version = action2.version.upPatch) + put(entityStore, action3) + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) + + // the action3 should be returned + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + } + it should "get action with updated field" in { implicit val tid = transid() @@ -1491,6 +1529,36 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "delete old action if its versions exceed the limit" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??")) + put(entityStore, action) + var version = action.version + + val limit = loadConfigOrThrow[Int](ConfigKeys.actionVersionLimit) + (1 until limit) foreach { _ => + version = version.upPatch + val actionN = action.copy(version = version) + put(entityStore, actionN) + } + + // the first version should be still there + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + } + // the first version should be deleted automatically + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + } + //// POST /actions/name it should "invoke an action with arguments, nonblocking" in { implicit val tid = transid() From 1b2088b7d93c0f8648d9cf1a661a3279ed9c9ec5 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Sat, 10 Oct 2020 11:00:44 +0800 Subject: [PATCH 05/29] Allow to replace specify version --- .../openwhisk/core/controller/Actions.scala | 7 +++- .../controller/test/ActionsApiTests.scala | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index e6d97bb1dd8..0b7fd2e99ff 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -231,7 +231,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case Success(_) => onComplete(WhiskActionVersionList.get(entityName, entityStore)) { case Success(result) => - val id = result.matchedDocId(None).getOrElse(entityName.toDocId) + val id = result.matchedDocId(content.version).getOrElse(entityName.toDocId) putEntity( WhiskAction, entityStore, @@ -685,6 +685,9 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with .map(_ ++ content.annotations) .getOrElse(action.annotations ++ content.annotations) + // if content provided a `version`, then new action should overwrite old entity in database + val newRev = content.version.map(_ => action.rev).getOrElse(DocRevision.empty) + WhiskAction( action.namespace, action.name, @@ -693,7 +696,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with limits, content.version getOrElse action.version.upPatch, content.publish getOrElse action.publish, - WhiskActionsApi.amendAnnotations(newAnnotations, exec, create = false)) + WhiskActionsApi.amendAnnotations(newAnnotations, exec, create = false)).revision[WhiskAction](newRev) } /** diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index 0781ec0408c..2ad8c57da81 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -1529,6 +1529,48 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "replace old action if version is provided" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) + put(entityStore, action) + val action2 = action.copy( + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS10, false)) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action2) + } + + // the first version should not be replaced + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // it should update based on the given version, action1 here, and replace it in database + val content2 = WhiskActionPut(version = Some(action.version), annotations = Some(Parameters("x", "X"))) + val action3 = + action.copy(annotations = action.annotations ++ content2.annotations ++ systemAnnotations(NODEJS10, false)) + Put(s"$collectionPath/${action.name}", content2) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action3) + } + + // the first version should be replaced with action3 + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + deleteAction(action2.docid) + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action3) + } + } + it should "delete old action if its versions exceed the limit" in { implicit val tid = transid() val action = WhiskAction(namespace, aname(), jsDefault("??")) From 171b9dccf6650c14af2ed3b261f2f1b8b06f4c37 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Sat, 10 Oct 2020 14:05:00 +0800 Subject: [PATCH 06/29] Fix bug --- .../files/whisks_design_document_for_entities_db_v2.1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index ecbdc5d9d4f..c535cd0b4ab 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -23,7 +23,7 @@ "reduce": "_count" }, "action-versions": { - "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var published = true;\n doc.annotations.forEach(function(anno) {\n if(anno[\"key\"] == \"publish\")\n published = anno[\"value\"]\n });\n \n var value = {\n namespace: doc.namespace,\n name: doc.name,\n _id: doc._id,\n version: doc.version,\n publish: published,\n };\n var versions = doc.version.split(\".\")\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var published = true;\n doc.annotations.forEach(function(anno) {\n if(anno[\"key\"] == \"publish\")\n published = anno[\"value\"]\n });\n \n var value = {\n namespace: doc.namespace,\n name: doc.name,\n id: doc._id,\n version: doc.version,\n publish: published,\n };\n var versions = doc.version.split(\".\")\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } } From e69240a31040bc0ad9b9fa1752ae689cff8e1910 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 12 Oct 2020 16:19:14 +0800 Subject: [PATCH 07/29] Implement action-version view for memoryDB and cosmosDB --- .../core/database/DocumentHandler.scala | 67 ++++++++++++------- .../cosmosdb/CosmosDBViewMapper.scala | 8 ++- .../database/memory/MemoryViewMapper.scala | 13 ++-- .../openwhisk/core/entity/WhiskAction.scala | 9 +-- 4 files changed, 64 insertions(+), 33 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala index f61aacc6a2d..bc17dee8af8 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala @@ -115,6 +115,28 @@ abstract class SimpleHandler extends DocumentHandler { * Key is an array which matches the view query key */ protected def createKey(ddoc: String, view: String, startKey: List[Any], js: JsObject): JsArray + + /** + * Finds and transforms annotation with matching key. + * + * @param js js object having annotations array + * @param key annotation key + * @param vtr transformer function to map annotation value + * @param default default value to use if no matching annotation found + * @return annotation value matching given key + */ + protected[database] def annotationValue[T](js: JsObject, key: String, vtr: JsValue => T, default: T): T = { + js.fields.get("annotations") match { + case Some(JsArray(e)) => + e.view + .map(_.asJsObject.getFields("key", "value")) + .collectFirst { + case Seq(JsString(`key`), v: JsValue) => vtr(v) //match annotation with given key + } + .getOrElse(default) + case _ => default + } + } } object ActivationHandler extends SimpleHandler { @@ -177,35 +199,15 @@ object ActivationHandler extends SimpleHandler { }, name) } - /** - * Finds and transforms annotation with matching key. - * - * @param js js object having annotations array - * @param key annotation key - * @param vtr transformer function to map annotation value - * @param default default value to use if no matching annotation found - * @return annotation value matching given key - */ - protected[database] def annotationValue[T](js: JsObject, key: String, vtr: JsValue => T, default: T): T = { - js.fields.get("annotations") match { - case Some(JsArray(e)) => - e.view - .map(_.asJsObject.getFields("key", "value")) - .collectFirst { - case Seq(JsString(`key`), v: JsValue) => vtr(v) //match annotation with given key - } - .getOrElse(default) - case _ => default - } - } - private def dropNull(fields: JsField*) = JsObject(fields.filter(_._2 != JsNull): _*) } object WhisksHandler extends SimpleHandler { val ROOT_NS = "rootns" + val FULL_NAME = "fullname" private val commonFields = Set("namespace", "name", "version", "publish", "annotations", "updated") private val actionFields = commonFields ++ Set("limits", "exec.binary") + private val actionVersionFields = commonFields ++ Set("_id") private val packageFields = commonFields ++ Set("binding") private val packagePublicFields = commonFields private val ruleFields = commonFields @@ -213,6 +215,7 @@ object WhisksHandler extends SimpleHandler { protected val supportedTables = Set( "whisks.v2.1.0/actions", + "whisks.v2.1.0/action-versions", "whisks.v2.1.0/packages", "whisks.v2.1.0/packages-public", "whisks.v2.1.0/rules", @@ -223,13 +226,20 @@ object WhisksHandler extends SimpleHandler { case Some(JsString(namespace)) => val ns = namespace.split(PATHSEP) val rootNS = if (ns.length > 1) ns(0) else namespace - JsObject((ROOT_NS, JsString(rootNS))) + js.fields.get("name") match { + case Some(JsString(name)) => + val fullName = s"$namespace$PATHSEP$name" + JsObject((ROOT_NS, JsString(rootNS)), (FULL_NAME, JsString(fullName))) + case _ => + JsObject((ROOT_NS, JsString(rootNS))) + } case _ => JsObject.empty } } override def fieldsRequiredForView(ddoc: String, view: String): Set[String] = view match { case "actions" => actionFields + case "action-versions" => actionVersionFields case "packages" => packageFields case "packages-public" => packagePublicFields case "rules" => ruleFields @@ -239,6 +249,7 @@ object WhisksHandler extends SimpleHandler { def computeView(ddoc: String, view: String, js: JsObject): JsObject = view match { case "actions" => computeActionView(js) + case "action-versions" => computeActionVersionsView(js) case "packages" => computePackageView(js) case "packages-public" => computePublicPackageView(js) case "rules" => computeRulesView(js) @@ -256,6 +267,7 @@ object WhisksHandler extends SimpleHandler { def getEntityTypeForDesignDoc(ddoc: String, view: String): String = view match { case "actions" => "action" + case "action-versions" => "action" case "rules" => "rule" case "triggers" => "trigger" case "packages" | "packages-public" => "package" @@ -288,6 +300,15 @@ object WhisksHandler extends SimpleHandler { val exec_binary = JsHelpers.getFieldPath(js, "exec", "binary") JsObject(base + ("exec" -> JsObject("binary" -> exec_binary.getOrElse(JsFalse)))) } + + private def computeActionVersionsView(js: JsObject): JsObject = { + val publish = annotationValue(js, "publish", { v => + v.convertTo[Boolean] + }, true) + + val base = js.fields.filterKeys(actionVersionFields).toMap + JsObject(base + ("publish" -> publish.toJson)) + } } object SubjectHandler extends DocumentHandler { diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala index 8e1e102f495..3bceb2ba452 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala @@ -26,6 +26,7 @@ import kamon.metric.MeasurementUnit import org.apache.openwhisk.common.{LogMarkerToken, TransactionId, WhiskInstants} import org.apache.openwhisk.core.database.ActivationHandler.NS_PATH import org.apache.openwhisk.core.database.WhisksHandler.ROOT_NS +import org.apache.openwhisk.core.database.WhisksHandler.FULL_NAME import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants.{alias, computed, deleted} import org.apache.openwhisk.core.database.{ ActivationHandler, @@ -143,6 +144,7 @@ private[cosmosdb] abstract class SimpleMapper extends CosmosDBViewMapper { private[cosmosdb] object WhisksViewMapper extends SimpleMapper { private val NS = "namespace" private val ROOT_NS_C = s"$computed.$ROOT_NS" + private val FULL_NAME_C = s"$computed.$FULL_NAME" private val TYPE = "entityType" private val UPDATED = "updated" private val PUBLISH = "publish" @@ -169,7 +171,8 @@ private[cosmosdb] object WhisksViewMapper extends SimpleMapper { viewConditions(ddoc, view).map(q => (s"${q._1} AND", q._2)).getOrElse((NOTHING, Nil)) val params = ("@entityType", entityType) :: ("@namespace", namespace) :: vcParams - val baseCondition = s"$vc r.$TYPE = @entityType AND (r.$NS = @namespace OR r.$ROOT_NS_C = @namespace)" + val baseCondition = + s"$vc r.$TYPE = @entityType AND (r.$NS = @namespace OR r.$ROOT_NS_C = @namespace OR r.$FULL_NAME_C = @namespace)" (startKey, endKey) match { case (_ :: Nil, _ :: `TOP` :: Nil) => @@ -194,7 +197,8 @@ private[cosmosdb] object WhisksViewMapper extends SimpleMapper { } override protected def orderByField(ddoc: String, view: String): String = view match { - case "actions" | "rules" | "triggers" | "packages" | "packages-public" if ddoc.startsWith("whisks") => + case "actions" | "action-versions" | "rules" | "triggers" | "packages" | "packages-public" + if ddoc.startsWith("whisks") => s"r.$UPDATED" case _ => throw UnsupportedView(s"$ddoc/$view") } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala index 4898e723ede..ab0a328ff4b 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala @@ -123,6 +123,7 @@ private object ActivationViewMapper extends MemoryViewMapper { private object WhisksViewMapper extends MemoryViewMapper { private val NS = "namespace" private val ROOT_NS = WhisksHandler.ROOT_NS + private val FULL_NAME = WhisksHandler.FULL_NAME private val TYPE = "entityType" private val UPDATED = "updated" private val PUBLISH = "publish" @@ -140,20 +141,23 @@ private object WhisksViewMapper extends MemoryViewMapper { val matchTypeAndView = equal(d, TYPE, entityType) && matchViewConditions(ddoc, view, d) val matchNS = equal(d, NS, startKey.head.asInstanceOf[String]) val matchRootNS = equal(c, ROOT_NS, startKey.head.asInstanceOf[String]) + val matchFullName = equal(c, FULL_NAME, startKey.head.asInstanceOf[String]) //Here ddocs for actions, rules and triggers use //namespace and namespace/packageName as first key val filterResult = (startKey, endKey) match { case (ns :: Nil, _ :: `TOP` :: Nil) => - (matchTypeAndView && matchNS) || (matchTypeAndView && matchRootNS) + (matchTypeAndView && matchNS) || (matchTypeAndView && matchRootNS) || (matchTypeAndView && matchFullName) case (ns :: (since: Number) :: Nil, _ :: `TOP` :: `TOP` :: Nil) => (matchTypeAndView && matchNS && gte(d, UPDATED, since)) || - (matchTypeAndView && matchRootNS && gte(d, UPDATED, since)) + (matchTypeAndView && matchRootNS && gte(d, UPDATED, since)) || + (matchTypeAndView && matchFullName && gte(d, UPDATED, since)) case (ns :: (since: Number) :: Nil, _ :: (upto: Number) :: `TOP` :: Nil) => (matchTypeAndView && matchNS && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) || - (matchTypeAndView && matchRootNS && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) + (matchTypeAndView && matchRootNS && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) || + (matchTypeAndView && matchFullName && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) case _ => throw UnsupportedQueryKeys(s"$ddoc/$view -> ($startKey, $endKey)") } @@ -177,7 +181,8 @@ private object WhisksViewMapper extends MemoryViewMapper { override def sort(ddoc: String, view: String, descending: Boolean, s: Seq[JsObject]): Seq[JsObject] = { view match { - case "actions" | "rules" | "triggers" | "packages" | "packages-public" if ddoc.startsWith("whisks") => + case "actions" | "action-versions" | "rules" | "triggers" | "packages" | "packages-public" + if ddoc.startsWith("whisks") => numericSort(s, descending, UPDATED) case _ => throw UnsupportedView(s"$ddoc/$view") } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 4eb44f48f6b..079ea836d20 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -361,7 +361,7 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer, publish: Boolean) object WhiskActionVersion { - val serdes = jsonFormat5(WhiskActionVersion.apply) + val serdes = jsonFormat(WhiskActionVersion.apply, "_id", "namespace", "name", "version", "publish") } case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versions: Map[SemVer, String]) { @@ -389,14 +389,15 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi implicit val logger: Logging = datastore.logging implicit val ec = datastore.executionContext - val key = List(action.fullPath.asString) + val startKey = List(action.fullPath.asString) + val endKey = List(action.fullPath.asString, WhiskQueries.TOP) cacheLookup( cacheKey(action), datastore .query( viewName, - startKey = key, - endKey = key, + startKey = startKey, + endKey = endKey, skip = 0, limit = 0, includeDocs = false, From 9f5c1d86c57dda27ecde02ec97eff9b7250e38ef Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Tue, 13 Oct 2020 09:50:44 +0800 Subject: [PATCH 08/29] Fix view bug --- .../files/whisks_design_document_for_entities_db_v2.1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index c535cd0b4ab..ecbdc5d9d4f 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -23,7 +23,7 @@ "reduce": "_count" }, "action-versions": { - "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var published = true;\n doc.annotations.forEach(function(anno) {\n if(anno[\"key\"] == \"publish\")\n published = anno[\"value\"]\n });\n \n var value = {\n namespace: doc.namespace,\n name: doc.name,\n id: doc._id,\n version: doc.version,\n publish: published,\n };\n var versions = doc.version.split(\".\")\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var published = true;\n doc.annotations.forEach(function(anno) {\n if(anno[\"key\"] == \"publish\")\n published = anno[\"value\"]\n });\n \n var value = {\n namespace: doc.namespace,\n name: doc.name,\n _id: doc._id,\n version: doc.version,\n publish: published,\n };\n var versions = doc.version.split(\".\")\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } } From 5d041c43cee56a6cc2b9a2814eba37a54809cc89 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Tue, 13 Oct 2020 11:11:47 +0800 Subject: [PATCH 09/29] Fix DocumentHandlerTests --- .../core/database/test/DocumentHandlerTests.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala index 725162b4a29..b59f7fd4691 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala @@ -28,7 +28,7 @@ import spray.json.DefaultJsonProtocol._ import spray.json._ import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.database.SubjectHandler.SubjectView -import org.apache.openwhisk.core.database.WhisksHandler.ROOT_NS +import org.apache.openwhisk.core.database.WhisksHandler.{FULL_NAME, ROOT_NS} import org.apache.openwhisk.core.database._ import org.apache.openwhisk.core.entity._ @@ -48,14 +48,18 @@ class DocumentHandlerTests extends FlatSpec with Matchers with ScalaFutures with it should "return JsObject when namespace is simple name" in { WhisksHandler.computedFields(JsObject(("namespace", JsString("foo")))) shouldBe JsObject((ROOT_NS, JsString("foo"))) - WhisksHandler.computedFields(newRule("foo").toDocumentRecord) shouldBe JsObject((ROOT_NS, JsString("foo"))) + WhisksHandler.computedFields(newRule("foo").toDocumentRecord) shouldBe JsObject( + (ROOT_NS, JsString("foo")), + (FULL_NAME, JsString("foo/foo"))) } it should "return JsObject when namespace is path" in { WhisksHandler.computedFields(JsObject(("namespace", JsString("foo/bar")))) shouldBe JsObject((ROOT_NS, JsString("foo"))) - WhisksHandler.computedFields(newRule("foo/bar").toDocumentRecord) shouldBe JsObject((ROOT_NS, JsString("foo"))) + WhisksHandler.computedFields(newRule("foo/bar").toDocumentRecord) shouldBe JsObject( + (ROOT_NS, JsString("foo")), + (FULL_NAME, JsString("foo/bar/foo"))) } private def newRule(ns: String): WhiskRule = { From a880b977b279e9e0c9a3e42d3be4dc273d48fddb Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Thu, 29 Oct 2020 14:36:02 +0800 Subject: [PATCH 10/29] Ensure data consistency --- .../openwhisk/core/entity/WhiskAction.scala | 18 ++++++++---- .../controller/test/ActionsApiTests.scala | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 079ea836d20..f824d1771cf 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -419,11 +419,13 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi }) } - def getMatchedDocId(action: FullyQualifiedEntityName, version: Option[SemVer], datastore: EntityStore)( - implicit transId: TransactionId, - ec: ExecutionContext): Future[Option[DocId]] = { - get(action, datastore, version.nonEmpty).map { res => - version match { + def getMatchedDocId( + action: FullyQualifiedEntityName, + version: Option[SemVer], + datastore: EntityStore, + tryAgain: Boolean = true)(implicit transId: TransactionId, ec: ExecutionContext): Future[Option[DocId]] = { + get(action, datastore, version.nonEmpty).flatMap { res => + val docId = version match { case Some(ver) => res.versions.get(ver).map(DocId(_)) case None if res.versions.nonEmpty => @@ -431,6 +433,12 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi case _ => None } + // there may be a chance that database is updated while cache is not, we need to invalidate cache and try again + if (docId.isEmpty && tryAgain) { + WhiskActionVersionList.removeId(cacheKey(action)) + getMatchedDocId(action, version, datastore, false) + } else + Future.successful(docId) } } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index 2ad8c57da81..94312cd7bfc 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -245,6 +245,34 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "ensure the data consistency" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // put entity to database directly with the cache in memory for version list untouched + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + Get(s"$collectionPath/${action.name}?version=0.0.2") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + } + it should "not get unpublished version" in { implicit val tid = transid() val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) From d1b2c54405f0ab87ec2f44931a56977e0e8e31dd Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Thu, 29 Oct 2020 15:56:46 +0800 Subject: [PATCH 11/29] Add deleteAll parameter for action#remove --- .../openwhisk/core/entity/WhiskAction.scala | 3 +- .../openwhisk/core/controller/Actions.scala | 6 +++- .../controller/test/ActionsApiTests.scala | 31 ++++++++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index f824d1771cf..7a2853cda25 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -378,7 +378,8 @@ case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versi } object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActionVersionList, DocInfo] { - lazy val viewName = WhiskQueries.entitiesView(collection = "action-versions").name + val collectionName = "action-versions" + lazy val viewName = WhiskQueries.entitiesView(collection = collectionName).name def cacheKey(action: FullyQualifiedEntityName): CacheKey = { CacheKey(action.fullPath.asString) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 0b7fd2e99ff..37d1eb0d1c3 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -372,7 +372,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 500 Internal Server Error */ override def remove(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { - parameter('version.as[SemVer] ?) { version => + parameter('version.as[SemVer] ?, 'deleteAll ? false) { (version, deleteAll) => onComplete(WhiskActionVersionList.get(entityName, entityStore)) { case Success(results) => version match { @@ -387,6 +387,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with WhiskActionVersionList.deleteCache(entityName) complete(OK, action) }) + case None if (!deleteAll && results.versions.size > 1) => + terminate( + Forbidden, + s"[DEL] entity version not provided, you need to specify deleteAll=true to delete all versions for action $entityName") case None => val fs = if (results.versions.isEmpty) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index 94312cd7bfc..eb09962aa9d 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -575,8 +575,13 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { response should be(action3) } - // it should delete all actions if version is not specified + // it should return 403 error when try to delete multi versions without specify deleteAll=true Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(Forbidden) + } + + // it should delete all actions if version is not specified and deleteAll is passed as true + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[WhiskAction] response should be(action3) @@ -951,7 +956,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs), - Some(action.limits.concurrency)))) + Some(action.limits.concurrency))), + Some(action.version)) // first request invalidates any previous entries and caches new result Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)(transid())) ~> check { @@ -1004,12 +1010,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version.upPatch, + action.version, action.publish, action.annotations ++ systemAnnotations(kind))) } stream.toString should include(s"entity exists, will try to update '$action'") - stream.toString should include(s"caching ${CacheKey(action.copy(version = action.version.upPatch))}") + stream.toString should include(s"caching ${CacheKey(action)}") stream.reset() // delete should invalidate cache for all versions @@ -1024,12 +1030,11 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version.upPatch, + action.version, action.publish, action.annotations ++ systemAnnotations(kind))) } stream.toString should include(s"invalidating ${CacheKey(action)}") - stream.toString should include(s"invalidating ${CacheKey(action.copy(version = action.version.upPatch))}") stream.reset() } } @@ -1343,7 +1348,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs), - Some(action.limits.concurrency)))) + Some(action.limits.concurrency))), + Some(action.version)) val name = action.name val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)") val expectedPutLog = @@ -1369,7 +1375,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version.upPatch, + action.version, action.publish, action.annotations ++ systemAnnotations(kind))) } @@ -1388,7 +1394,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version.upPatch, + action.version, action.publish, action.annotations ++ systemAnnotations(kind))) } @@ -1410,7 +1416,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(actionOldSchema.limits.timeout), Some(actionOldSchema.limits.memory), Some(actionOldSchema.limits.logs), - Some(actionOldSchema.limits.concurrency)))) + Some(actionOldSchema.limits.concurrency))), + Some(actionOldSchema.version)) val expectedPutLog = Seq(s"uploading attachment '[\\w-/:]+' of document 'id: ${actionOldSchema.namespace}/${actionOldSchema.name}") .mkString("(?s).*") @@ -1436,7 +1443,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { actionNewSchema.exec, actionOldSchema.parameters, actionOldSchema.limits, - actionOldSchema.version.upPatch, + actionOldSchema.version, actionOldSchema.publish, actionOldSchema.annotations ++ systemAnnotations(NODEJS, create = false))) } @@ -1461,7 +1468,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { actionNewSchema.exec, actionOldSchema.parameters, actionOldSchema.limits, - actionOldSchema.version.upPatch, + actionOldSchema.version, actionOldSchema.publish, actionOldSchema.annotations ++ systemAnnotations(NODEJS, create = false))) } From 2b1c4e8c8997171669a8550c9e77037d9521a707 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Thu, 29 Oct 2020 16:18:11 +0800 Subject: [PATCH 12/29] Add deleteOld parameter for action#create --- .../org/apache/openwhisk/core/controller/Actions.scala | 6 +++++- .../openwhisk/core/controller/test/ActionsApiTests.scala | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 37d1eb0d1c3..36cfe615fdc 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -220,7 +220,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 500 Internal Server Error */ override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { - parameter('overwrite ? false) { overwrite => + parameter('overwrite ? false, 'deleteOld ? false) { (overwrite, deleteOld) => entity(as[WhiskActionPut]) { content => val request = content.resolve(user.namespace) val checkAdditionalPrivileges = entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap { @@ -230,6 +230,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with onComplete(checkAdditionalPrivileges) { case Success(_) => onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + case Success(result) if (result.versions.size >= actionMaxVersionLimit && !deleteOld) => + terminate( + Forbidden, + s"[PUT] entity has ${result.versions.size} versions exist which exceed $actionMaxVersionLimit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") case Success(result) => val id = result.matchedDocId(content.version).getOrElse(entityName.toDocId) putEntity( diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index eb09962aa9d..2d8ae622535 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -1628,6 +1628,10 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(Forbidden) + } + + Put(s"$collectionPath/${action.name}?deleteOld=true", content) ~> Route.seal(routes(creds)) ~> check { status should be(OK) } // the first version should be deleted automatically From e030b4b4cb3b8e8047a4f5c51baef2309e7b2d77 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Tue, 3 Nov 2020 14:14:46 +0800 Subject: [PATCH 13/29] Revert "Allow to replace specify version" This reverts commit 49396bf7b1ac837140448073b7d335541827d61c. --- .../openwhisk/core/controller/Actions.scala | 7 +--- .../controller/test/ActionsApiTests.scala | 42 ------------------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 36cfe615fdc..ce6e3779480 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -235,7 +235,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with Forbidden, s"[PUT] entity has ${result.versions.size} versions exist which exceed $actionMaxVersionLimit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") case Success(result) => - val id = result.matchedDocId(content.version).getOrElse(entityName.toDocId) + val id = result.matchedDocId(None).getOrElse(entityName.toDocId) putEntity( WhiskAction, entityStore, @@ -693,9 +693,6 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with .map(_ ++ content.annotations) .getOrElse(action.annotations ++ content.annotations) - // if content provided a `version`, then new action should overwrite old entity in database - val newRev = content.version.map(_ => action.rev).getOrElse(DocRevision.empty) - WhiskAction( action.namespace, action.name, @@ -704,7 +701,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with limits, content.version getOrElse action.version.upPatch, content.publish getOrElse action.publish, - WhiskActionsApi.amendAnnotations(newAnnotations, exec, create = false)).revision[WhiskAction](newRev) + WhiskActionsApi.amendAnnotations(newAnnotations, exec, create = false)) } /** diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index 2d8ae622535..c5d5551e8c4 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -1564,48 +1564,6 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } - it should "replace old action if version is provided" in { - implicit val tid = transid() - val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) - val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) - put(entityStore, action) - val action2 = action.copy( - parameters = content.parameters.get, - version = action.version.upPatch, - annotations = action.annotations ++ systemAnnotations(NODEJS10, false)) - Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - checkWhiskEntityResponse(response, action2) - } - - // the first version should not be replaced - Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action) - } - - // it should update based on the given version, action1 here, and replace it in database - val content2 = WhiskActionPut(version = Some(action.version), annotations = Some(Parameters("x", "X"))) - val action3 = - action.copy(annotations = action.annotations ++ content2.annotations ++ systemAnnotations(NODEJS10, false)) - Put(s"$collectionPath/${action.name}", content2) ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - checkWhiskEntityResponse(response, action3) - } - - // the first version should be replaced with action3 - Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { - deleteAction(action.docid) - deleteAction(action2.docid) - status should be(OK) - val response = responseAs[WhiskAction] - checkWhiskEntityResponse(response, action3) - } - } - it should "delete old action if its versions exceed the limit" in { implicit val tid = transid() val action = WhiskAction(namespace, aname(), jsDefault("??")) From 4d0e220f6126295dc49891e7a4270b6d233f8974 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Tue, 3 Nov 2020 15:28:01 +0800 Subject: [PATCH 14/29] add default version feature --- ...esign_document_for_entities_db_v2.1.0.json | 2 +- .../openwhisk/core/entity/WhiskAction.scala | 29 ++++++++++++++----- .../openwhisk/core/controller/Actions.scala | 4 +-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index ecbdc5d9d4f..f4398633f33 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -23,7 +23,7 @@ "reduce": "_count" }, "action-versions": { - "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var published = true;\n doc.annotations.forEach(function(anno) {\n if(anno[\"key\"] == \"publish\")\n published = anno[\"value\"]\n });\n \n var value = {\n namespace: doc.namespace,\n name: doc.name,\n _id: doc._id,\n version: doc.version,\n publish: published,\n };\n var versions = doc.version.split(\".\")\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n _id: doc.namespace + \"/\" + doc.name + \"/default\",\n namespace: doc.namespace,\n name: doc.name,\n id: doc._id,\n version: doc.version\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 7a2853cda25..7cbed613eaa 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -361,14 +361,19 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer, publish: Boolean) object WhiskActionVersion { - val serdes = jsonFormat(WhiskActionVersion.apply, "_id", "namespace", "name", "version", "publish") + val serdes = jsonFormat(WhiskActionVersion.apply, "id", "namespace", "name", "version", "publish") } -case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versions: Map[SemVer, String]) { +case class WhiskActionVersionList(namespace: EntityPath, + name: EntityName, + versions: Map[SemVer, String], + defaultVersion: Option[String]) { def matchedDocId(version: Option[SemVer]): Option[DocId] = { version match { case Some(ver) => versions.get(ver).map(DocId(_)) + case None if defaultVersion.nonEmpty => + versions.get(SemVer(defaultVersion.get)).map(DocId(_)) case None if versions.nonEmpty => Some(DocId(versions.maxBy(_._1)._2)) case _ => @@ -385,7 +390,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi CacheKey(action.fullPath.asString) } - def get(action: FullyQualifiedEntityName, datastore: EntityStore, fetchAll: Boolean = true)( + def get(action: FullyQualifiedEntityName, datastore: EntityStore, fromCache: Boolean = true)( implicit transId: TransactionId): Future[WhiskActionVersionList] = { implicit val logger: Logging = datastore.logging implicit val ec = datastore.executionContext @@ -401,7 +406,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi endKey = endKey, skip = 0, limit = 0, - includeDocs = false, + includeDocs = true, descending = false, reduce = false, stale = StaleParameter.No) @@ -411,13 +416,19 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi } val mappings = values .map(WhiskActionVersion.serdes.read(_)) - .filter(_.publish || fetchAll) .map { actionVersion => (actionVersion.version, actionVersion.id) } .toMap - WhiskActionVersionList(action.namespace.toPath, action.name, mappings) - }) + val defaultVersion = if (result.nonEmpty) { + val doc = result.head.fields.getOrElse("doc", JsNull) + if (doc != JsNull) doc.asJsObject.fields.get("default").map(_.convertTo[String]) + else + None + } else None + WhiskActionVersionList(action.namespace.toPath, action.name, mappings, defaultVersion) + }, + fromCache) } def getMatchedDocId( @@ -425,10 +436,12 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi version: Option[SemVer], datastore: EntityStore, tryAgain: Boolean = true)(implicit transId: TransactionId, ec: ExecutionContext): Future[Option[DocId]] = { - get(action, datastore, version.nonEmpty).flatMap { res => + get(action, datastore).flatMap { res => val docId = version match { case Some(ver) => res.versions.get(ver).map(DocId(_)) + case None if res.defaultVersion.nonEmpty => + res.versions.get(SemVer(res.defaultVersion.get)).map(DocId(_)) case None if res.versions.nonEmpty => Some(DocId(res.versions.maxBy(_._1)._2)) case _ => diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index ce6e3779480..90f8a08ff1e 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -229,7 +229,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with onComplete(checkAdditionalPrivileges) { case Success(_) => - onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { case Success(result) if (result.versions.size >= actionMaxVersionLimit && !deleteOld) => terminate( Forbidden, @@ -377,7 +377,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with */ override def remove(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { parameter('version.as[SemVer] ?, 'deleteAll ? false) { (version, deleteAll) => - onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { case Success(results) => version match { case Some(_) => From af0d7c2f24d2bc9c6ff4148e390b78d2190c0777 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Wed, 11 Nov 2020 15:00:31 +0800 Subject: [PATCH 15/29] Some updates: 1. provide a migrate script to copy old actions to new docs with new id 2. use a separate entity to save the default version for action --- .../openwhisk/core/entity/WhiskAction.scala | 74 +++-- .../openwhisk/core/entity/WhiskEntity.scala | 15 +- .../openwhisk/core/controller/Actions.scala | 155 ++++++---- .../openwhisk/core/controller/Packages.scala | 4 + .../test/scala/common/WskCliOperations.scala | 2 + .../src/test/scala/common/WskOperations.scala | 2 + .../scala/common/rest/WskRestOperations.scala | 9 +- .../controller/test/ActionsApiTests.scala | 267 +++++++++++++----- .../test/ControllerTestCommon.scala | 16 +- .../scala/system/basic/WskActionTests.scala | 24 +- tools/migrate/migrate_to_new_id.py | 54 ++++ 11 files changed, 453 insertions(+), 169 deletions(-) create mode 100644 tools/migrate/migrate_to_new_id.py diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 7cbed613eaa..4585b336f2e 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -34,8 +34,10 @@ import org.apache.openwhisk.core.database.{ ArtifactStore, CacheChangeNotification, DocumentFactory, + EvictionPolicy, MultipleReadersSingleWriterCache, - StaleParameter + StaleParameter, + WriteTime } import org.apache.openwhisk.core.entity.Attachments._ import org.apache.openwhisk.core.entity.types.EntityStore @@ -358,22 +360,22 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, } -case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer, publish: Boolean) +case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer) object WhiskActionVersion { - val serdes = jsonFormat(WhiskActionVersion.apply, "id", "namespace", "name", "version", "publish") + val serdes = jsonFormat(WhiskActionVersion.apply, "id", "namespace", "name", "version") } case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, versions: Map[SemVer, String], - defaultVersion: Option[String]) { + defaultVersion: Option[SemVer]) { def matchedDocId(version: Option[SemVer]): Option[DocId] = { version match { case Some(ver) => - versions.get(ver).map(DocId(_)) + Some(DocId(s"$namespace/$name@$ver")) case None if defaultVersion.nonEmpty => - versions.get(SemVer(defaultVersion.get)).map(DocId(_)) + Some(DocId(s"$namespace/$name@${defaultVersion.get}")) case None if versions.nonEmpty => Some(DocId(versions.maxBy(_._1)._2)) case _ => @@ -383,8 +385,10 @@ case class WhiskActionVersionList(namespace: EntityPath, } object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActionVersionList, DocInfo] { + override val evictionPolicy: EvictionPolicy = WriteTime val collectionName = "action-versions" lazy val viewName = WhiskQueries.entitiesView(collection = collectionName).name + implicit val serdes = jsonFormat(WhiskActionVersionList.apply, "namespace", "name", "versions", "defaultVersion") def cacheKey(action: FullyQualifiedEntityName): CacheKey = { CacheKey(action.fullPath.asString) @@ -421,38 +425,30 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi } .toMap val defaultVersion = if (result.nonEmpty) { - val doc = result.head.fields.getOrElse("doc", JsNull) - if (doc != JsNull) doc.asJsObject.fields.get("default").map(_.convertTo[String]) - else - None + result.head.fields.get("doc") match { + case Some(value) => Try { value.asJsObject.fields.get("default").map(_.convertTo[SemVer]) } getOrElse None + case None => None + } } else None WhiskActionVersionList(action.namespace.toPath, action.name, mappings, defaultVersion) }, fromCache) } - def getMatchedDocId( - action: FullyQualifiedEntityName, - version: Option[SemVer], - datastore: EntityStore, - tryAgain: Boolean = true)(implicit transId: TransactionId, ec: ExecutionContext): Future[Option[DocId]] = { - get(action, datastore).flatMap { res => - val docId = version match { - case Some(ver) => - res.versions.get(ver).map(DocId(_)) + def getMatchedDocId(action: FullyQualifiedEntityName, version: Option[SemVer], datastore: EntityStore)( + implicit transId: TransactionId, + ec: ExecutionContext): Future[Option[DocId]] = { + get(action, datastore).map { res => + version match { + case Some(_) => + Some(DocId(action.copy(version = version).asString)) case None if res.defaultVersion.nonEmpty => - res.versions.get(SemVer(res.defaultVersion.get)).map(DocId(_)) + Some(DocId(action.copy(version = res.defaultVersion).asString)) case None if res.versions.nonEmpty => Some(DocId(res.versions.maxBy(_._1)._2)) case _ => None } - // there may be a chance that database is updated while cache is not, we need to invalidate cache and try again - if (docId.isEmpty && tryAgain) { - WhiskActionVersionList.removeId(cacheKey(action)) - getMatchedDocId(action, version, datastore, false) - } else - Future.successful(docId) } } @@ -465,6 +461,32 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi } } +object WhiskActionDefaultVersion extends DocumentFactory[WhiskActionDefaultVersion] { + import WhiskActivation.instantSerdes + implicit val serdes = jsonFormat(WhiskActionDefaultVersion.apply, "namespace", "name", "default", "updated") +} + +case class WhiskActionDefaultVersion(namespace: EntityPath, + override val name: EntityName, + default: Option[SemVer] = None, + override val updated: Instant = WhiskEntity.currentMillis()) + extends WhiskEntity(name, "action-default-version") { + + /** + * The representation as JSON, e.g. for REST calls. Does not include id/rev. + */ + override def toJson: JsObject = WhiskActionDefaultVersion.serdes.write(this).asJsObject + + /** + * Gets unique document identifier for the document. + */ + override def docid: DocId = new DocId(namespace + EntityPath.PATHSEP + name + "/default") + + override val version: SemVer = SemVer() + override val publish: Boolean = true + override val annotations: Parameters = Parameters() +} + object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[WhiskAction] with DefaultJsonProtocol { import WhiskActivation.instantSerdes diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala index da867214918..7f39b783955 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala @@ -125,13 +125,14 @@ object WhiskEntity { object WhiskDocumentReader extends DocumentReader { override def read[A](ma: Manifest[A], value: JsValue) = { val doc = ma.runtimeClass match { - case x if x == classOf[WhiskAction] => WhiskAction.serdes.read(value) - case x if x == classOf[WhiskActionMetaData] => WhiskActionMetaData.serdes.read(value) - case x if x == classOf[WhiskPackage] => WhiskPackage.serdes.read(value) - case x if x == classOf[WhiskActivation] => WhiskActivation.serdes.read(value) - case x if x == classOf[WhiskTrigger] => WhiskTrigger.serdes.read(value) - case x if x == classOf[WhiskRule] => WhiskRule.serdes.read(value) - case _ => throw DocumentUnreadable(Messages.corruptedEntity) + case x if x == classOf[WhiskAction] => WhiskAction.serdes.read(value) + case x if x == classOf[WhiskActionMetaData] => WhiskActionMetaData.serdes.read(value) + case x if x == classOf[WhiskPackage] => WhiskPackage.serdes.read(value) + case x if x == classOf[WhiskActivation] => WhiskActivation.serdes.read(value) + case x if x == classOf[WhiskTrigger] => WhiskTrigger.serdes.read(value) + case x if x == classOf[WhiskRule] => WhiskRule.serdes.read(value) + case x if x == classOf[WhiskActionDefaultVersion] => WhiskActionDefaultVersion.serdes.read(value) + case _ => throw DocumentUnreadable(Messages.corruptedEntity) } value.asJsObject.fields.get("entityType").foreach { case JsString(entityType) if (doc.entityType != entityType) => diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 90f8a08ff1e..2403f599b6f 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -220,53 +220,80 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 500 Internal Server Error */ override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { - parameter('overwrite ? false, 'deleteOld ? false) { (overwrite, deleteOld) => - entity(as[WhiskActionPut]) { content => - val request = content.resolve(user.namespace) - val checkAdditionalPrivileges = entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap { - case _ => entitlementProvider.check(user, content.exec) - } - - onComplete(checkAdditionalPrivileges) { - case Success(_) => - onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { - case Success(result) if (result.versions.size >= actionMaxVersionLimit && !deleteOld) => - terminate( - Forbidden, - s"[PUT] entity has ${result.versions.size} versions exist which exceed $actionMaxVersionLimit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") - case Success(result) => - val id = result.matchedDocId(None).getOrElse(entityName.toDocId) - putEntity( - WhiskAction, - entityStore, - id, - true, - update(user, request) _, - () => { - make(user, entityName, request) - }, - postProcess = Some { action: WhiskAction => - // delete oldest version when created successfully - if (result.versions.size >= actionMaxVersionLimit) { - val id = result.versions.minBy(_._1)._2 - WhiskAction.get(entityStore, DocId(id)) flatMap { entity => - WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) - } andThen { - case _ => - WhiskActionVersionList.deleteCache(entityName) - } - } else { + parameter('overwrite ? false, 'deleteOld ? false, 'defaultVersion.as[String] ? "") { + (overwrite, deleteOld, defaultVersion) => + entity(as[WhiskActionPut]) { content => + Try { + SemVer(defaultVersion) + } match { + case Success(version) => + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { + case Success(result) if (result.versions.keys.toVector.contains(version)) => + val dv = WhiskActionDefaultVersion(entityName.path, entityName.name, Some(version)) + putEntity( + WhiskActionDefaultVersion, + entityStore, + dv.docid, + true, + (old: WhiskActionDefaultVersion) => + Future.successful(dv.revision[WhiskActionDefaultVersion](old.rev)), + () => Future.successful(dv), + postProcess = Some { version: WhiskActionDefaultVersion => WhiskActionVersionList.deleteCache(entityName) - } - complete(OK, action) - }) - case Failure(f) => - terminate(InternalServerError) - } - case Failure(f) => - super.handleEntitlementFailure(f) + complete(OK, version) + }) + case Success(_) => + terminate(Forbidden, s"[PUT] entity doesn't has version $version") + case Failure(_) => + terminate(InternalServerError) + } + case Failure(_) => + val request = content.resolve(user.namespace) + val checkAdditionalPrivileges = entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap { + case _ => entitlementProvider.check(user, content.exec) + } + + onComplete(checkAdditionalPrivileges) { + case Success(_) => + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { + case Success(result) if (result.versions.size >= actionMaxVersionLimit && !deleteOld) => + terminate( + Forbidden, + s"[PUT] entity has ${result.versions.size} versions exist which exceed $actionMaxVersionLimit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") + case Success(result) => + val id = result.matchedDocId(None).getOrElse(entityName.toDocId) + putEntity( + WhiskAction, + entityStore, + id, + true, + update(user, request) _, + () => { + make(user, entityName, request) + }, + postProcess = Some { action: WhiskAction => + // delete oldest version when created successfully + if (result.versions.size >= actionMaxVersionLimit) { + val id = result.versions.minBy(_._1)._2 + WhiskAction.get(entityStore, DocId(id)) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) + } andThen { + case _ => + WhiskActionVersionList.deleteCache(entityName) + } + } else { + WhiskActionVersionList.deleteCache(entityName) + } + complete(OK, action) + }) + case Failure(f) => + terminate(InternalServerError) + } + case Failure(f) => + super.handleEntitlementFailure(f) + } + } } - } } } @@ -388,10 +415,14 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with docId, (a: WhiskAction) => Future.successful({}), postProcess = Some { action: WhiskAction => + // when default version is deleted or all versions are deleted, delete the default version entity + if (version == results.defaultVersion || results.versions.size == 1) + deleteDefaultVersion( + WhiskActionDefaultVersion(entityName.path, entityName.name, results.defaultVersion)) WhiskActionVersionList.deleteCache(entityName) complete(OK, action) }) - case None if (!deleteAll && results.versions.size > 1) => + case None if !deleteAll && results.versions.size > 1 => terminate( Forbidden, s"[DEL] entity version not provided, you need to specify deleteAll=true to delete all versions for action $entityName") @@ -412,6 +443,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case _ => WhiskActionVersionList .deleteCache(entityName) // invalidate version list cache after all deletion completed + deleteDefaultVersion(WhiskActionDefaultVersion( + entityName.path, + entityName.name, + results.defaultVersion)) // delete default version entity since all versions are deleted } onComplete(deleteFuture) { @@ -443,6 +478,19 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with } } + private def deleteDefaultVersion(defaultVersion: WhiskActionDefaultVersion)(implicit transid: TransactionId): Unit = { + WhiskActionDefaultVersion.get(entityStore, defaultVersion.docid) map { entity => + WhiskActionDefaultVersion.del(entityStore, defaultVersion.docid.asDocInfo(entity.rev)) andThen { + case Success(_) => + logging.info(this, s"[DEL] default version for ${defaultVersion.fullyQualifiedName(false)} is deleted") + case Failure(t) => + logging.error( + this, + s"[DEL] failed to delete default version for ${defaultVersion.fullyQualifiedName(false)}, error: $t") + } + } + } + /** Checks for package binding case. we don't want to allow get for a package binding in shared package */ private def fetchEntity(entityName: FullyQualifiedEntityName, env: Option[Parameters], @@ -499,10 +547,17 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with */ override def fetch(user: Identity, entityName: FullyQualifiedEntityName, env: Option[Parameters])( implicit transid: TransactionId) = { - parameter('code ? true, 'version.as[SemVer] ?) { (code, version) => - //check if execute only is enabled, and if there is a discrepancy between the current user's namespace - //and that of the entity we are trying to fetch - if (executeOnly && user.namespace.name != entityName.namespace) { + parameter('code ? true, 'version.as[SemVer] ?, 'showVersions ? false) { (code, version, showVersions) => + if (showVersions) { + onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + case Success(res) => + complete(OK, res) + case Failure(t) => + terminate(Forbidden, forbiddenGetAction(entityName.path.asString)) + } + //check if execute only is enabled, and if there is a discrepancy between the current user's namespace + //and that of the entity we are trying to fetch + } else if (executeOnly && user.namespace.name != entityName.namespace) { terminate(Forbidden, forbiddenGetAction(entityName.path.asString)) } else { fetchEntity(entityName, env, code, version) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala index 68a19512dc1..1e42c98d774 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala @@ -162,6 +162,10 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { case _ => list.foreach { action => WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) + val version = WhiskActionDefaultVersion(action.namespace, action.name, None) + WhiskActionDefaultVersion.get(entityStore, version.docid) foreach { versionWithRevision => + WhiskActionDefaultVersion.del(entityStore, versionWithRevision.docinfo) + } } } flatMap { _ => Future.successful({}) diff --git a/tests/src/test/scala/common/WskCliOperations.scala b/tests/src/test/scala/common/WskCliOperations.scala index 45e16a38cb1..2d91aa1ac83 100644 --- a/tests/src/test/scala/common/WskCliOperations.scala +++ b/tests/src/test/scala/common/WskCliOperations.scala @@ -206,6 +206,8 @@ class CliActionOperations(override val wsk: RunCliCmd) update: Boolean = false, web: Option[String] = None, websecure: Option[String] = None, + deleteOld: Boolean = true, + defaultVersion: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult = { val params = Seq(noun, if (!update) "create" else "update", "--auth", wp.authKey, fqn(name)) ++ { artifact map { Seq(_) } getOrElse Seq.empty diff --git a/tests/src/test/scala/common/WskOperations.scala b/tests/src/test/scala/common/WskOperations.scala index 7782745e670..0bee6675248 100644 --- a/tests/src/test/scala/common/WskOperations.scala +++ b/tests/src/test/scala/common/WskOperations.scala @@ -245,6 +245,8 @@ trait ActionOperations extends DeleteFromCollectionOperations with ListOrGetFrom update: Boolean = false, web: Option[String] = None, websecure: Option[String] = None, + deleteOld: Boolean = true, + defaultVersion: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult def invoke(name: String, diff --git a/tests/src/test/scala/common/rest/WskRestOperations.scala b/tests/src/test/scala/common/rest/WskRestOperations.scala index 35e91463c6e..42c63d0ac9d 100644 --- a/tests/src/test/scala/common/rest/WskRestOperations.scala +++ b/tests/src/test/scala/common/rest/WskRestOperations.scala @@ -204,7 +204,7 @@ trait RestDeleteFromCollectionOperations extends DeleteFromCollectionOperations override def delete(name: String, expectedExitCode: Int = OK.intValue)(implicit wp: WskProps): RestResult = { val (ns, entityName) = getNamespaceEntityName(name) val path = Path(s"$basePath/namespaces/$ns/$noun/$entityName") - val resp = requestEntity(DELETE, path)(wp) + val resp = requestEntity(DELETE, path, Map("deleteAll" -> "true"))(wp) val rr = new RestResult(resp.status, getTransactionId(resp), getRespData(resp)) validateStatusCode(expectedExitCode, rr.statusCode.intValue) rr @@ -275,6 +275,8 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) update: Boolean = false, web: Option[String] = None, websecure: Option[String] = None, + deleteOld: Boolean = true, + defaultVersion: Option[String] = None, expectedExitCode: Int = OK.intValue)(implicit wp: WskProps): RestResult = { val (namespace, actionName) = getNamespaceEntityName(name) @@ -371,8 +373,11 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) } val path = Path(s"$basePath/namespaces/$namespace/$noun/$actionName") + val paramemters = + Map("overwrite" -> update.toString, "deleteOld" -> deleteOld.toString) ++ defaultVersion.map(version => + ("defaultVersion" -> version)) val resp = - if (update) requestEntity(PUT, path, Map("overwrite" -> "true"), Some(JsObject(body).toString)) + if (update) requestEntity(PUT, path, paramemters, Some(JsObject(body).toString)) else requestEntity(PUT, path, body = Some(JsObject(body).toString)) val rr = new RestResult(resp.status, getTransactionId(resp), getRespData(resp)) validateStatusCode(expectedExitCode, rr.statusCode.intValue) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index c5d5551e8c4..b12b9efd773 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -245,68 +245,6 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } - it should "ensure the data consistency" in { - implicit val tid = transid() - val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) - put(entityStore, action) - - Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action) - } - - // put entity to database directly with the cache in memory for version list untouched - val action2 = action.copy(version = action.version.upPatch) - put(entityStore, action2) - - Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action) - } - - Get(s"$collectionPath/${action.name}?version=0.0.2") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action2) - } - } - - it should "not get unpublished version" in { - implicit val tid = transid() - val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) - put(entityStore, action) - - Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action) - } - - val action2 = action.copy(version = action.version.upPatch, annotations = Parameters("publish", JsFalse)) - put(entityStore, action2) - WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) - - // the action2 should not be returned - Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action) - } - - val action3 = action.copy(version = action2.version.upPatch) - put(entityStore, action3) - WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) - - // the action3 should be returned - Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[WhiskAction] - response should be(action3) - } - } - it should "get action with updated field" in { implicit val tid = transid() @@ -592,6 +530,104 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "delete default version if all versions of an action are deleted" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + val action3 = action2.copy(version = action2.version.upPatch) + put(entityStore, action3) + + val content = WhiskActionPut() + + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.2", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + + val response = responseAs[WhiskActionDefaultVersion] + checkWhiskEntityResponse( + response, + WhiskActionDefaultVersion(action.namespace, action.name, Some(action2.version))) + } + + // it should return action@0.0.2 + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + // delete all actions + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + // default version should be deleted and latest version should be used + put(entityStore, action) + put(entityStore, action2) + put(entityStore, action3) + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + // it should delete all actions if version is not specified and deleteAll is passed as true + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + } + + it should "delete default version if related version of the action is deleted" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + val action3 = action2.copy(version = action2.version.upPatch) + put(entityStore, action3) + + val content = WhiskActionPut() + + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.1", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + + val response = responseAs[WhiskActionDefaultVersion] + checkWhiskEntityResponse(response, WhiskActionDefaultVersion(action.namespace, action.name, Some(action.version))) + } + + // it should return action@0.0.1 + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // delete action@0.0.1 + Delete(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // default version should be deleted and latest version should be used + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + deleteAction(action2.docid) + deleteAction(action3.docid) + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + } + it should "report NotFound for delete non existent action" in { implicit val tid = transid() Delete(s"$collectionPath/xyz") ~> Route.seal(routes(creds)) ~> check { @@ -956,8 +992,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs), - Some(action.limits.concurrency))), - Some(action.version)) + Some(action.limits.concurrency)))) // first request invalidates any previous entries and caches new result Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)(transid())) ~> check { @@ -1010,16 +1045,16 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version, + action.version.upPatch, action.publish, action.annotations ++ systemAnnotations(kind))) } stream.toString should include(s"entity exists, will try to update '$action'") - stream.toString should include(s"caching ${CacheKey(action)}") + stream.toString should include(s"caching ${CacheKey(action.copy(version = action.version.upPatch))}") stream.reset() // delete should invalidate cache for all versions - Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)(transid())) ~> check { + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1030,7 +1065,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version, + action.version.upPatch, action.publish, action.annotations ++ systemAnnotations(kind))) } @@ -1348,8 +1383,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(action.limits.timeout), Some(action.limits.memory), Some(action.limits.logs), - Some(action.limits.concurrency))), - Some(action.version)) + Some(action.limits.concurrency)))) val name = action.name val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")", "\\)") val expectedPutLog = @@ -1375,7 +1409,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version, + action.version.upPatch, action.publish, action.annotations ++ systemAnnotations(kind))) } @@ -1383,7 +1417,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { stream.reset() // delete should invalidate cache - Delete(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check { + Delete(s"$collectionPath/$name?deleteAll=true") ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1394,7 +1428,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.exec, action.parameters, action.limits, - action.version, + action.version.upPatch, action.publish, action.annotations ++ systemAnnotations(kind))) } @@ -1416,8 +1450,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(actionOldSchema.limits.timeout), Some(actionOldSchema.limits.memory), Some(actionOldSchema.limits.logs), - Some(actionOldSchema.limits.concurrency))), - Some(actionOldSchema.version)) + Some(actionOldSchema.limits.concurrency)))) val expectedPutLog = Seq(s"uploading attachment '[\\w-/:]+' of document 'id: ${actionOldSchema.namespace}/${actionOldSchema.name}") .mkString("(?s).*") @@ -1443,7 +1476,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { actionNewSchema.exec, actionOldSchema.parameters, actionOldSchema.limits, - actionOldSchema.version, + actionOldSchema.version.upPatch, actionOldSchema.publish, actionOldSchema.annotations ++ systemAnnotations(NODEJS, create = false))) } @@ -1457,7 +1490,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { response.fields("activationId") should not be None } - Delete(s"$collectionPath/${actionOldSchema.name}") ~> Route.seal(routes(creds)) ~> check { + Delete(s"$collectionPath/${actionOldSchema.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1468,7 +1501,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { actionNewSchema.exec, actionOldSchema.parameters, actionOldSchema.limits, - actionOldSchema.version, + actionOldSchema.version.upPatch, actionOldSchema.publish, actionOldSchema.annotations ++ systemAnnotations(NODEJS, create = false))) } @@ -1509,6 +1542,84 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "update default version only if default version is provided" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"), ActionLimits()) + val content = WhiskActionPut( + Some(jsDefault("_")), + Some(Parameters("x", "X")), + Some( + ActionLimitsOption( + Some(TimeLimit(TimeLimit.MAX_DURATION)), + Some(MemoryLimit(MemoryLimit.MAX_MEMORY)), + Some(LogLimit(LogLimit.MAX_LOGSIZE)), + Some(ConcurrencyLimit(ConcurrencyLimit.MAX_CONCURRENT))))) + put(entityStore, action) + + val action2 = action.copy( + exec = content.exec.get, + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS10, create = false), + limits = ActionLimits( + content.limits.get.timeout.get, + content.limits.get.memory.get, + content.limits.get.logs.get, + content.limits.get.concurrency.get)) + + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + + response.updated should not be action.updated + checkWhiskEntityResponse(response, action2) + } + + // get version 0.0.2 as no default version is set(the latest one will be choose) + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action2) + } + + // set default version to 0.0.1 + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.1", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskActionDefaultVersion] + checkWhiskEntityResponse(response, WhiskActionDefaultVersion(action.namespace, action.name, Some(action.version))) + } + + // get version 0.0.1 + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + deleteAction(action2.docid) + deleteActionDefaultVersion(DocId(s"${action.fullyQualifiedName(false)}/default")) + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + } + + it should "reject update default version if it doesn't exist" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"), ActionLimits()) + val content = WhiskActionPut( + Some(jsDefault("_")), + Some(Parameters("x", "X")), + Some( + ActionLimitsOption( + Some(TimeLimit(TimeLimit.MAX_DURATION)), + Some(MemoryLimit(MemoryLimit.MAX_MEMORY)), + Some(LogLimit(LogLimit.MAX_LOGSIZE)), + Some(ConcurrencyLimit(ConcurrencyLimit.MAX_CONCURRENT))))) + put(entityStore, action) + + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.2", content) ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + status should be(Forbidden) + } + } + it should "update action parameters with a put" in { implicit val tid = transid() val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala index 7431f12d308..03eae259bff 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala @@ -89,10 +89,11 @@ protected trait ControllerTestCommon // Used to ignore `updated` field because timestamp is not known before inserting into the DB // If you use this method, test case that checks timestamp must be added val r = response match { - case whiskAction: WhiskAction => whiskAction.copy(updated = expected.updated) - case whiskActionMetaData: WhiskActionMetaData => whiskActionMetaData.copy(updated = expected.updated) - case whiskTrigger: WhiskTrigger => whiskTrigger.copy(updated = expected.updated) - case whiskPackage: WhiskPackage => whiskPackage.copy(updated = expected.updated) + case whiskAction: WhiskAction => whiskAction.copy(updated = expected.updated) + case defaultVersion: WhiskActionDefaultVersion => defaultVersion.copy(updated = expected.updated) + case whiskActionMetaData: WhiskActionMetaData => whiskActionMetaData.copy(updated = expected.updated) + case whiskTrigger: WhiskTrigger => whiskTrigger.copy(updated = expected.updated) + case whiskPackage: WhiskPackage => whiskPackage.copy(updated = expected.updated) } r should be(expected) } @@ -124,6 +125,13 @@ protected trait ControllerTestCommon Await.result(activationStore.get(activationId, context), timeout) } + def deleteActionDefaultVersion(doc: DocId)(implicit transid: TransactionId) = { + Await.result(WhiskActionDefaultVersion.get(entityStore, doc) flatMap { doc => + logging.debug(this, s"deleting ${doc.docinfo}") + WhiskActionDefaultVersion.del(entityStore, doc.docinfo) + }, dbOpTimeout) + } + def storeActivation( activation: WhiskActivation, isBlockingActivation: Boolean, diff --git a/tests/src/test/scala/system/basic/WskActionTests.scala b/tests/src/test/scala/system/basic/WskActionTests.scala index 4d25d798bcc..ae092f62043 100644 --- a/tests/src/test/scala/system/basic/WskActionTests.scala +++ b/tests/src/test/scala/system/basic/WskActionTests.scala @@ -53,13 +53,13 @@ class WskActionTests extends TestHelpers with WskTestHelpers with JsHelpers with } it should "save multi versions for an action and invoke them" in withAssetCleaner(wskprops) { (wp, assetHelper) => - val name = "hello" + val name = "multiVersion" assetHelper.withCleaner(wsk.action, name) { (action, _) => action.create(name, Some(TestUtils.getTestActionFilename("hello.js"))) action.create(name, Some(TestUtils.getTestActionFilename("echo.js"))) } - // invoke the latest version + // invoke the default version var run = wsk.action.invoke(name, Map("payload" -> "world".toJson)) withActivation(wsk.activation, run) { activation => activation.response.status shouldBe "success" @@ -83,6 +83,26 @@ class WskActionTests extends TestHelpers with WskTestHelpers with JsHelpers with expectedExitCode = NotFound.intValue) } + it should "invoke the default version of an action if version parameter is not provided" in withAssetCleaner(wskprops) { + (wp, assetHelper) => + val name = "defaultVersion" + assetHelper.withCleaner(wsk.action, name) { (action, _) => + action.create(name, Some(TestUtils.getTestActionFilename("hello.js"))) + action.create(name, Some(TestUtils.getTestActionFilename("echo.js"))) + } + + // set the default version + wsk.action.create(name, None, defaultVersion = Some("0.0.2")) + + // invoke the default version + val run = wsk.action.invoke(name, Map("payload" -> "world".toJson)) + withActivation(wsk.activation, run) { activation => + activation.response.status shouldBe "success" + activation.response.result shouldBe Some(JsObject("payload" -> "world".toJson)) + activation.logs.get.mkString(" ") shouldBe empty + } + } + it should "invoke an action returning a promise" in withAssetCleaner(wskprops) { (wp, assetHelper) => val name = "hello promise" assetHelper.withCleaner(wsk.action, name) { (action, _) => diff --git a/tools/migrate/migrate_to_new_id.py b/tools/migrate/migrate_to_new_id.py new file mode 100644 index 00000000000..4afcfe034f1 --- /dev/null +++ b/tools/migrate/migrate_to_new_id.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +"""Python script to delete old Activations. +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +""" + +import argparse +import couchdb.client +import requests +from urllib import quote_plus + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Utility to migrate actions with new ids.") + parser.add_argument("--dbUrl", required=True, help="Server URL of the database, that has to be cleaned of old activations. E.g. 'https://xxx:yyy@domain.couch.com:443'") + parser.add_argument("--dbName", required=True, help="Name of the Database of the actions to be migration.") + parser.add_argument("--docsPerRequest", type=int, default=200, help="Number of documents handled on each CouchDb Request. Default is 200.") + + args = parser.parse_args() + + db = couchdb.client.Server(args.dbUrl)[args.dbName] + start = 0 + actions = db.view("whisks.v2.1.0/actions", skip=start, limit=args.docsPerRequest, reduce=False) + + while len(actions) != 0: + for action in actions: + actionName = "%s/%s" % (action.value["namespace"], action.value["name"]) + newId = "%s@%s" % (actionName, action.value["version"]) + # this action is using old style id, copy it with new id which append `@version` to it + if(action.id != newId): + print("Copy %s to %s:...........\n" % (action.id, newId)) + url = "%s/%s/%s" % (args.dbUrl, args.dbName, quote_plus(actionName)) + headers = {"Content-Type": "application/json", "Destination": newId} + res = requests.request("COPY", url, headers = headers) + print("Copying result is %s\n" % res.content) + else: + print("Action %s is already using new style id, skip it\n" % action.id) + + start = start + args.docsPerRequest + actions = db.view("whisks.v2.1.0/actions", skip=start, limit=args.docsPerRequest, reduce=False) From 5e4bc9f6a70ad955687c10008358d1579bb27a04 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Wed, 11 Nov 2020 17:45:36 +0800 Subject: [PATCH 16/29] Fix format style --- tools/migrate/migrate_to_new_id.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/migrate/migrate_to_new_id.py b/tools/migrate/migrate_to_new_id.py index 4afcfe034f1..1392a911d77 100644 --- a/tools/migrate/migrate_to_new_id.py +++ b/tools/migrate/migrate_to_new_id.py @@ -29,7 +29,7 @@ parser.add_argument("--dbUrl", required=True, help="Server URL of the database, that has to be cleaned of old activations. E.g. 'https://xxx:yyy@domain.couch.com:443'") parser.add_argument("--dbName", required=True, help="Name of the Database of the actions to be migration.") parser.add_argument("--docsPerRequest", type=int, default=200, help="Number of documents handled on each CouchDb Request. Default is 200.") - + args = parser.parse_args() db = couchdb.client.Server(args.dbUrl)[args.dbName] @@ -39,7 +39,7 @@ while len(actions) != 0: for action in actions: actionName = "%s/%s" % (action.value["namespace"], action.value["name"]) - newId = "%s@%s" % (actionName, action.value["version"]) + newId = "%s@%s" % (actionName, action.value["version"]) # this action is using old style id, copy it with new id which append `@version` to it if(action.id != newId): print("Copy %s to %s:...........\n" % (action.id, newId)) From 4dc81470f814e71cf1838cfe1356329187018987 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Fri, 13 Nov 2020 18:08:01 +0800 Subject: [PATCH 17/29] Fix test --- .../core/database/DocumentHandler.scala | 25 ++++++++++--------- .../scala/system/basic/WskActionTests.scala | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala index bc17dee8af8..d82fdc4bcf6 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala @@ -96,13 +96,17 @@ abstract class SimpleHandler extends DocumentHandler { provider: DocumentProvider)(implicit transid: TransactionId, ec: ExecutionContext): Future[Seq[JsObject]] = { //Query result from CouchDB have below object structure with actual result in `value` key //So transform the result to confirm to that structure - val viewResult = JsObject( - "id" -> js.fields("_id"), - "key" -> createKey(ddoc, view, startKey, js), - "value" -> computeView(ddoc, view, js)) + val value = computeView(ddoc, view, js) + val viewResult = JsObject("id" -> js.fields("_id"), "key" -> createKey(ddoc, view, startKey, js), "value" -> value) - val result = if (includeDocs) JsObject(viewResult.fields + ("doc" -> js)) else viewResult - Future.successful(Seq(result)) + if (includeDocs) value.fields.get("_id") match { + case Some(JsString(id)) if id != js.fields("_id") => + provider.get(DocId(id)).map { doc => + Seq(JsObject(viewResult.fields + ("doc" -> doc.getOrElse(js)))) + } + case _ => + Future.successful(Seq(JsObject(viewResult.fields + ("doc" -> js)))) + } else Future.successful(Seq(viewResult)) } /** @@ -207,7 +211,7 @@ object WhisksHandler extends SimpleHandler { val FULL_NAME = "fullname" private val commonFields = Set("namespace", "name", "version", "publish", "annotations", "updated") private val actionFields = commonFields ++ Set("limits", "exec.binary") - private val actionVersionFields = commonFields ++ Set("_id") + private val actionVersionFields = commonFields ++ Set("_id", "id") private val packageFields = commonFields ++ Set("binding") private val packagePublicFields = commonFields private val ruleFields = commonFields @@ -302,12 +306,9 @@ object WhisksHandler extends SimpleHandler { } private def computeActionVersionsView(js: JsObject): JsObject = { - val publish = annotationValue(js, "publish", { v => - v.convertTo[Boolean] - }, true) - val base = js.fields.filterKeys(actionVersionFields).toMap - JsObject(base + ("publish" -> publish.toJson)) + val defaultId = js.fields("namespace") + "/" + js.fields("name") + "/default" + JsObject(base + ("_id" -> JsString(defaultId), "id" -> js.fields("_id"))) } } diff --git a/tests/src/test/scala/system/basic/WskActionTests.scala b/tests/src/test/scala/system/basic/WskActionTests.scala index ae092f62043..5a988eead33 100644 --- a/tests/src/test/scala/system/basic/WskActionTests.scala +++ b/tests/src/test/scala/system/basic/WskActionTests.scala @@ -92,7 +92,7 @@ class WskActionTests extends TestHelpers with WskTestHelpers with JsHelpers with } // set the default version - wsk.action.create(name, None, defaultVersion = Some("0.0.2")) + wsk.action.create(name, Some(TestUtils.getTestActionFilename("hello.js")), defaultVersion = Some("0.0.2")) // invoke the default version val run = wsk.action.invoke(name, Map("payload" -> "world".toJson)) From ef1711c7ad2fe9dc3924cf1731c0380357895799 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 16 Nov 2020 15:20:41 +0800 Subject: [PATCH 18/29] Remove some useless code --- ...esign_document_for_entities_db_v2.1.0.json | 2 +- .../core/database/DocumentHandler.scala | 4 +-- .../openwhisk/core/entity/WhiskAction.scala | 28 ++++++++----------- .../openwhisk/core/controller/Actions.scala | 14 +++++----- .../test/scala/common/WskCliOperations.scala | 1 - .../src/test/scala/common/WskOperations.scala | 1 - .../scala/common/rest/WskRestOperations.scala | 6 +--- .../scala/system/basic/WskActionTests.scala | 20 ------------- 8 files changed, 23 insertions(+), 53 deletions(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index f4398633f33..3a4acfc8e81 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -23,7 +23,7 @@ "reduce": "_count" }, "action-versions": { - "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n _id: doc.namespace + \"/\" + doc.name + \"/default\",\n namespace: doc.namespace,\n name: doc.name,\n id: doc._id,\n version: doc.version\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n _id: doc.namespace + \"/\" + doc.name + \"/default\",\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala index d82fdc4bcf6..67e41e5bac3 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala @@ -211,7 +211,7 @@ object WhisksHandler extends SimpleHandler { val FULL_NAME = "fullname" private val commonFields = Set("namespace", "name", "version", "publish", "annotations", "updated") private val actionFields = commonFields ++ Set("limits", "exec.binary") - private val actionVersionFields = commonFields ++ Set("_id", "id") + private val actionVersionFields = commonFields ++ Set("_id") private val packageFields = commonFields ++ Set("binding") private val packagePublicFields = commonFields private val ruleFields = commonFields @@ -308,7 +308,7 @@ object WhisksHandler extends SimpleHandler { private def computeActionVersionsView(js: JsObject): JsObject = { val base = js.fields.filterKeys(actionVersionFields).toMap val defaultId = js.fields("namespace") + "/" + js.fields("name") + "/default" - JsObject(base + ("_id" -> JsString(defaultId), "id" -> js.fields("_id"))) + JsObject(base + ("_id" -> JsString(defaultId))) } } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 4585b336f2e..b519d79c615 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -360,15 +360,9 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, } -case class WhiskActionVersion(id: String, namespace: EntityPath, name: EntityName, version: SemVer) - -object WhiskActionVersion { - val serdes = jsonFormat(WhiskActionVersion.apply, "id", "namespace", "name", "version") -} - case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, - versions: Map[SemVer, String], + versions: List[SemVer], defaultVersion: Option[SemVer]) { def matchedDocId(version: Option[SemVer]): Option[DocId] = { version match { @@ -377,7 +371,7 @@ case class WhiskActionVersionList(namespace: EntityPath, case None if defaultVersion.nonEmpty => Some(DocId(s"$namespace/$name@${defaultVersion.get}")) case None if versions.nonEmpty => - Some(DocId(versions.maxBy(_._1)._2)) + Some(DocId(s"$namespace/$name@${versions.max}")) case _ => None } @@ -418,19 +412,21 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi val values = result.map { row => row.fields("value").asJsObject() } - val mappings = values - .map(WhiskActionVersion.serdes.read(_)) - .map { actionVersion => - (actionVersion.version, actionVersion.id) - } - .toMap + val versions = values.map { value => + Try { value.fields.get("version").map(_.convertTo[SemVer]) } getOrElse None + } + val defaultVersion = if (result.nonEmpty) { result.head.fields.get("doc") match { case Some(value) => Try { value.asJsObject.fields.get("default").map(_.convertTo[SemVer]) } getOrElse None case None => None } } else None - WhiskActionVersionList(action.namespace.toPath, action.name, mappings, defaultVersion) + WhiskActionVersionList( + action.namespace.toPath, + action.name, + versions.filter(_.nonEmpty).map(_.get), + defaultVersion) }, fromCache) } @@ -445,7 +441,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi case None if res.defaultVersion.nonEmpty => Some(DocId(action.copy(version = res.defaultVersion).asString)) case None if res.versions.nonEmpty => - Some(DocId(res.versions.maxBy(_._1)._2)) + Some(DocId(action.copy(version = Some(res.versions.max)).asString)) case _ => None } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 2403f599b6f..e4fc0e4decd 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -228,7 +228,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with } match { case Success(version) => onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { - case Success(result) if (result.versions.keys.toVector.contains(version)) => + case Success(result) if (result.versions.contains(version)) => val dv = WhiskActionDefaultVersion(entityName.path, entityName.name, Some(version)) putEntity( WhiskActionDefaultVersion, @@ -274,7 +274,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with postProcess = Some { action: WhiskAction => // delete oldest version when created successfully if (result.versions.size >= actionMaxVersionLimit) { - val id = result.versions.minBy(_._1)._2 + val id = entityName.copy(version = Some(result.versions.min)).asString WhiskAction.get(entityStore, DocId(id)) flatMap { entity => WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) } andThen { @@ -433,12 +433,12 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) }) else - results.versions.values - .map { id => - WhiskAction.get(entityStore, DocId(id)) flatMap { entity => - WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) - } + results.versions.map { version => + val id = entityName.copy(version = Some(version)).asString + WhiskAction.get(entityStore, DocId(id)) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) } + } val deleteFuture = Future.sequence(fs).andThen { case _ => WhiskActionVersionList diff --git a/tests/src/test/scala/common/WskCliOperations.scala b/tests/src/test/scala/common/WskCliOperations.scala index 2d91aa1ac83..7be210a2848 100644 --- a/tests/src/test/scala/common/WskCliOperations.scala +++ b/tests/src/test/scala/common/WskCliOperations.scala @@ -207,7 +207,6 @@ class CliActionOperations(override val wsk: RunCliCmd) web: Option[String] = None, websecure: Option[String] = None, deleteOld: Boolean = true, - defaultVersion: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult = { val params = Seq(noun, if (!update) "create" else "update", "--auth", wp.authKey, fqn(name)) ++ { artifact map { Seq(_) } getOrElse Seq.empty diff --git a/tests/src/test/scala/common/WskOperations.scala b/tests/src/test/scala/common/WskOperations.scala index 0bee6675248..217dc5c061a 100644 --- a/tests/src/test/scala/common/WskOperations.scala +++ b/tests/src/test/scala/common/WskOperations.scala @@ -246,7 +246,6 @@ trait ActionOperations extends DeleteFromCollectionOperations with ListOrGetFrom web: Option[String] = None, websecure: Option[String] = None, deleteOld: Boolean = true, - defaultVersion: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult def invoke(name: String, diff --git a/tests/src/test/scala/common/rest/WskRestOperations.scala b/tests/src/test/scala/common/rest/WskRestOperations.scala index 42c63d0ac9d..093507269c1 100644 --- a/tests/src/test/scala/common/rest/WskRestOperations.scala +++ b/tests/src/test/scala/common/rest/WskRestOperations.scala @@ -276,7 +276,6 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) web: Option[String] = None, websecure: Option[String] = None, deleteOld: Boolean = true, - defaultVersion: Option[String] = None, expectedExitCode: Int = OK.intValue)(implicit wp: WskProps): RestResult = { val (namespace, actionName) = getNamespaceEntityName(name) @@ -373,11 +372,8 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) } val path = Path(s"$basePath/namespaces/$namespace/$noun/$actionName") - val paramemters = - Map("overwrite" -> update.toString, "deleteOld" -> deleteOld.toString) ++ defaultVersion.map(version => - ("defaultVersion" -> version)) val resp = - if (update) requestEntity(PUT, path, paramemters, Some(JsObject(body).toString)) + if (update) requestEntity(PUT, path, Map("overwrite" -> "true"), Some(JsObject(body).toString)) else requestEntity(PUT, path, body = Some(JsObject(body).toString)) val rr = new RestResult(resp.status, getTransactionId(resp), getRespData(resp)) validateStatusCode(expectedExitCode, rr.statusCode.intValue) diff --git a/tests/src/test/scala/system/basic/WskActionTests.scala b/tests/src/test/scala/system/basic/WskActionTests.scala index 5a988eead33..7694e5d9b3f 100644 --- a/tests/src/test/scala/system/basic/WskActionTests.scala +++ b/tests/src/test/scala/system/basic/WskActionTests.scala @@ -83,26 +83,6 @@ class WskActionTests extends TestHelpers with WskTestHelpers with JsHelpers with expectedExitCode = NotFound.intValue) } - it should "invoke the default version of an action if version parameter is not provided" in withAssetCleaner(wskprops) { - (wp, assetHelper) => - val name = "defaultVersion" - assetHelper.withCleaner(wsk.action, name) { (action, _) => - action.create(name, Some(TestUtils.getTestActionFilename("hello.js"))) - action.create(name, Some(TestUtils.getTestActionFilename("echo.js"))) - } - - // set the default version - wsk.action.create(name, Some(TestUtils.getTestActionFilename("hello.js")), defaultVersion = Some("0.0.2")) - - // invoke the default version - val run = wsk.action.invoke(name, Map("payload" -> "world".toJson)) - withActivation(wsk.activation, run) { activation => - activation.response.status shouldBe "success" - activation.response.result shouldBe Some(JsObject("payload" -> "world".toJson)) - activation.logs.get.mkString(" ") shouldBe empty - } - } - it should "invoke an action returning a promise" in withAssetCleaner(wskprops) { (wp, assetHelper) => val name = "hello promise" assetHelper.withCleaner(wsk.action, name) { (action, _) => From 5278fdc261a572f3b97055aea98633284dc4b5ac Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 16 Nov 2020 17:33:09 +0800 Subject: [PATCH 19/29] Fix bug --- .../org/apache/openwhisk/core/entity/WhiskAction.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index b519d79c615..8222974c6f6 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -422,11 +422,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi case None => None } } else None - WhiskActionVersionList( - action.namespace.toPath, - action.name, - versions.filter(_.nonEmpty).map(_.get), - defaultVersion) + WhiskActionVersionList(action.path, action.name, versions.filter(_.nonEmpty).map(_.get), defaultVersion) }, fromCache) } From f2b348a589b5f92c95d825988aba1989f4741a0f Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Sat, 8 May 2021 13:31:32 +0800 Subject: [PATCH 20/29] Fix rebase error --- .../openwhisk/core/entity/WhiskAction.scala | 21 ++++++------------- .../openwhisk/core/controller/Actions.scala | 3 ++- .../v2/InvokerHealthManager.scala | 1 + 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 8222974c6f6..546198c5d3c 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -365,12 +365,12 @@ case class WhiskActionVersionList(namespace: EntityPath, versions: List[SemVer], defaultVersion: Option[SemVer]) { def matchedDocId(version: Option[SemVer]): Option[DocId] = { - version match { - case Some(ver) => + (version, defaultVersion) match { + case (Some(ver), _) => Some(DocId(s"$namespace/$name@$ver")) - case None if defaultVersion.nonEmpty => - Some(DocId(s"$namespace/$name@${defaultVersion.get}")) - case None if versions.nonEmpty => + case (None, Some(default)) => + Some(DocId(s"$namespace/$name@$default")) + case (None, None) if versions.nonEmpty => Some(DocId(s"$namespace/$name@${versions.max}")) case _ => None @@ -431,16 +431,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi implicit transId: TransactionId, ec: ExecutionContext): Future[Option[DocId]] = { get(action, datastore).map { res => - version match { - case Some(_) => - Some(DocId(action.copy(version = version).asString)) - case None if res.defaultVersion.nonEmpty => - Some(DocId(action.copy(version = res.defaultVersion).asString)) - case None if res.versions.nonEmpty => - Some(DocId(action.copy(version = Some(res.versions.max)).asString)) - case _ => - None - } + res.matchedDocId(version) } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index e4fc0e4decd..f41b14c6443 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -451,7 +451,8 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with onComplete(deleteFuture) { case Success(entities) => - complete(OK, entities.last) + val versions = entities.map(_.version).mkString(",") + complete(OK, s"[DEL] entity ${entities.last.fullyQualifiedName(false)} for versions $versions") case Failure(t: NoDocumentException) => logging.debug(this, s"[DEL] entity does not exist") terminate(NotFound) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala index 54bd174189f..b10bfd13bd1 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala @@ -324,6 +324,7 @@ object InvokerHealthManager { action.rev, healthActionIdentity, ActivationId.generate(), + docInfo.id, ControllerInstanceId("health"), blocking = false, content = None)) From becdb0282d3c53ad4b7ad66ee4da37ef71a86202 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Sat, 8 May 2021 14:49:41 +0800 Subject: [PATCH 21/29] Fix tests error --- .../openwhisk/core/controller/test/ActionsApiTests.scala | 3 +-- .../scheduler/grpc/test/ActivationServiceImplTests.scala | 1 + .../core/scheduler/grpc/test/CommonVariable.scala | 9 +++++++-- .../queue/test/ElasticSearchDurationCheckerTests.scala | 5 +++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index b12b9efd773..a54cbf760eb 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -1560,7 +1560,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { exec = content.exec.get, parameters = content.parameters.get, version = action.version.upPatch, - annotations = action.annotations ++ systemAnnotations(NODEJS10, create = false), + annotations = action.annotations ++ systemAnnotations(NODEJS, create = false), limits = ActionLimits( content.limits.get.timeout.get, content.limits.get.memory.get, @@ -1671,7 +1671,6 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse(response, action3) - >>>>>>> Implement action versioning } } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala index 75c913e9e34..0d15aa901ac 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala @@ -78,6 +78,7 @@ class ActivationServiceImplTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + DocId(s"$testEntityPath/$testEntityName@0.0.1"), ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala index 04f0e818013..f397dff00ad 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala @@ -29,12 +29,17 @@ trait CommonVariable { val testEntityName = EntityName(testAction) val testDocRevision = DocRevision("1-test-revision") val testContainerId = "fakeContainerId" - val semVer = SemVer(0, 1, 1) + val semVer = SemVer(0, 0, 1) val testVersion = Some(semVer) val testFQN = FullyQualifiedEntityName(testEntityPath, testEntityName, testVersion) val testExec = CodeExecAsString(RuntimeManifest("nodejs:14", ImageName("testImage")), "testCode", None) val testExecMetadata = CodeExecMetaDataAsString(testExec.manifest, entryPoint = testExec.entryPoint) val testActionMetaData = - WhiskActionMetaData(testEntityPath, testEntityName, testExecMetadata, version = semVer) + WhiskActionMetaData( + testEntityPath, + testEntityName, + DocId(s"$testEntityPath/$testEntityName@$semVer"), + testExecMetadata, + version = semVer) } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala index 8b30d60cfee..76090281713 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala @@ -139,6 +139,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(namespace), EntityName(actionName), + DocId(s"$namespace/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency)) @@ -210,6 +211,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace/$packageName"), EntityName(actionName), + DocId(s"$namespace/$packageName/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency)) @@ -288,6 +290,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace/$boundPackageName"), EntityName(actionName), + DocId(s"$namespace/$boundPackageName/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency), binding = Some(EntityPath(s"$namespace/$packageName"))) @@ -365,6 +368,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace"), EntityName(actionName), + DocId(s"$namespace/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency)) @@ -385,6 +389,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace/$boundPackageName"), EntityName(actionName), + DocId(s"$namespace/$boundPackageName/$actionName@0.0.1"), jsMetaData(Some("jsMain"), false), limits = actionLimits(actionMem, concurrency), binding = Some(EntityPath(s"${namespace}/${packageName}"))) From 6fe1d7f36de76878361a0f7880a537e53e390a41 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Sat, 8 May 2021 17:00:49 +0800 Subject: [PATCH 22/29] Fix tests --- .../scala/org/apache/openwhisk/core/controller/Actions.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index f41b14c6443..e4fc0e4decd 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -451,8 +451,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with onComplete(deleteFuture) { case Success(entities) => - val versions = entities.map(_.version).mkString(",") - complete(OK, s"[DEL] entity ${entities.last.fullyQualifiedName(false)} for versions $versions") + complete(OK, entities.last) case Failure(t: NoDocumentException) => logging.debug(this, s"[DEL] entity does not exist") terminate(NotFound) From a3939ed010069bffa0e33d2cfb4155668a7be1b2 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Mon, 10 May 2021 11:35:41 +0800 Subject: [PATCH 23/29] Use version mappings to get doc id from an version --- ...esign_document_for_entities_db_v2.1.0.json | 2 +- .../core/database/DocumentHandler.scala | 4 +- .../openwhisk/core/entity/WhiskAction.scala | 21 +++-- .../openwhisk/core/controller/Actions.scala | 79 +++++++++---------- tools/migrate/migrate_to_new_id.py | 4 +- 5 files changed, 58 insertions(+), 52 deletions(-) diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index 3a4acfc8e81..f1a899d2123 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -23,7 +23,7 @@ "reduce": "_count" }, "action-versions": { - "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n _id: doc.namespace + \"/\" + doc.name + \"/default\",\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n _id: doc.namespace + \"/\" + doc.name + \"/default\",\n namespace: doc.namespace,\n name: doc.name,\n docId: doc._id,\n version: doc.version\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala index 67e41e5bac3..ff63468285c 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala @@ -211,7 +211,7 @@ object WhisksHandler extends SimpleHandler { val FULL_NAME = "fullname" private val commonFields = Set("namespace", "name", "version", "publish", "annotations", "updated") private val actionFields = commonFields ++ Set("limits", "exec.binary") - private val actionVersionFields = commonFields ++ Set("_id") + private val actionVersionFields = commonFields ++ Set("_id", "docId") private val packageFields = commonFields ++ Set("binding") private val packagePublicFields = commonFields private val ruleFields = commonFields @@ -308,7 +308,7 @@ object WhisksHandler extends SimpleHandler { private def computeActionVersionsView(js: JsObject): JsObject = { val base = js.fields.filterKeys(actionVersionFields).toMap val defaultId = js.fields("namespace") + "/" + js.fields("name") + "/default" - JsObject(base + ("_id" -> JsString(defaultId))) + JsObject(base + ("_id" -> JsString(defaultId), "docId" -> js.fields("_id"))) } } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index 546198c5d3c..526f7d83fb2 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -362,16 +362,16 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, case class WhiskActionVersionList(namespace: EntityPath, name: EntityName, - versions: List[SemVer], + versionMappings: Map[SemVer, DocId], defaultVersion: Option[SemVer]) { def matchedDocId(version: Option[SemVer]): Option[DocId] = { (version, defaultVersion) match { case (Some(ver), _) => - Some(DocId(s"$namespace/$name@$ver")) + versionMappings.get(ver) case (None, Some(default)) => - Some(DocId(s"$namespace/$name@$default")) - case (None, None) if versions.nonEmpty => - Some(DocId(s"$namespace/$name@${versions.max}")) + versionMappings.get(default) + case (None, None) if versionMappings.nonEmpty => + Some(versionMappings.maxBy(_._1)._2) case _ => None } @@ -413,7 +413,14 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi row.fields("value").asJsObject() } val versions = values.map { value => - Try { value.fields.get("version").map(_.convertTo[SemVer]) } getOrElse None + val docId = value.fields.get("docId").map(_.compactPrint) + val version = Try { value.fields.get("version").map(_.convertTo[SemVer]) } getOrElse None + (version, docId) match { + case (Some(ver), Some(id)) => + Some((ver, DocId(id))) + case _ => + None + } } val defaultVersion = if (result.nonEmpty) { @@ -422,7 +429,7 @@ object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActi case None => None } } else None - WhiskActionVersionList(action.path, action.name, versions.filter(_.nonEmpty).map(_.get), defaultVersion) + WhiskActionVersionList(action.path, action.name, versions.filter(_.nonEmpty).map(_.get).toMap, defaultVersion) }, fromCache) } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index e4fc0e4decd..c4c2218ef82 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -222,32 +222,32 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { parameter('overwrite ? false, 'deleteOld ? false, 'defaultVersion.as[String] ? "") { (overwrite, deleteOld, defaultVersion) => - entity(as[WhiskActionPut]) { content => - Try { - SemVer(defaultVersion) - } match { - case Success(version) => - onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { - case Success(result) if (result.versions.contains(version)) => - val dv = WhiskActionDefaultVersion(entityName.path, entityName.name, Some(version)) - putEntity( - WhiskActionDefaultVersion, - entityStore, - dv.docid, - true, - (old: WhiskActionDefaultVersion) => - Future.successful(dv.revision[WhiskActionDefaultVersion](old.rev)), - () => Future.successful(dv), - postProcess = Some { version: WhiskActionDefaultVersion => - WhiskActionVersionList.deleteCache(entityName) - complete(OK, version) - }) - case Success(_) => - terminate(Forbidden, s"[PUT] entity doesn't has version $version") - case Failure(_) => - terminate(InternalServerError) - } - case Failure(_) => + Try { + SemVer(defaultVersion) + } match { + case Success(version) => + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { + case Success(result) if (result.versionMappings.contains(version)) => + val dv = WhiskActionDefaultVersion(entityName.path, entityName.name, Some(version)) + putEntity( + WhiskActionDefaultVersion, + entityStore, + dv.docid, + true, + (old: WhiskActionDefaultVersion) => + Future.successful(dv.revision[WhiskActionDefaultVersion](old.rev)), + () => Future.successful(dv), + postProcess = Some { version: WhiskActionDefaultVersion => + WhiskActionVersionList.deleteCache(entityName) + complete(OK, version) + }) + case Success(_) => + terminate(Forbidden, s"[PUT] entity doesn't has version $version") + case Failure(_) => + terminate(InternalServerError) + } + case Failure(_) => + entity(as[WhiskActionPut]) { content => val request = content.resolve(user.namespace) val checkAdditionalPrivileges = entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap { case _ => entitlementProvider.check(user, content.exec) @@ -256,10 +256,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with onComplete(checkAdditionalPrivileges) { case Success(_) => onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { - case Success(result) if (result.versions.size >= actionMaxVersionLimit && !deleteOld) => + case Success(result) if (result.versionMappings.size >= actionMaxVersionLimit && !deleteOld) => terminate( Forbidden, - s"[PUT] entity has ${result.versions.size} versions exist which exceed $actionMaxVersionLimit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") + s"[PUT] entity has ${result.versionMappings.size} versions exist which exceed maximum limit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") case Success(result) => val id = result.matchedDocId(None).getOrElse(entityName.toDocId) putEntity( @@ -273,10 +273,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with }, postProcess = Some { action: WhiskAction => // delete oldest version when created successfully - if (result.versions.size >= actionMaxVersionLimit) { - val id = entityName.copy(version = Some(result.versions.min)).asString - WhiskAction.get(entityStore, DocId(id)) flatMap { entity => - WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) + if (result.versionMappings.size >= actionMaxVersionLimit) { + val docid = result.versionMappings.minBy(_._1)._2 + WhiskAction.get(entityStore, docid) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (docid.id, entity.rev.rev)).map(_ => entity) } andThen { case _ => WhiskActionVersionList.deleteCache(entityName) @@ -292,7 +292,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case Failure(f) => super.handleEntitlementFailure(f) } - } + } } } } @@ -416,27 +416,26 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with (a: WhiskAction) => Future.successful({}), postProcess = Some { action: WhiskAction => // when default version is deleted or all versions are deleted, delete the default version entity - if (version == results.defaultVersion || results.versions.size == 1) + if (version == results.defaultVersion || results.versionMappings.size == 1) deleteDefaultVersion( WhiskActionDefaultVersion(entityName.path, entityName.name, results.defaultVersion)) WhiskActionVersionList.deleteCache(entityName) complete(OK, action) }) - case None if !deleteAll && results.versions.size > 1 => + case None if !deleteAll && results.versionMappings.size > 1 => terminate( Forbidden, s"[DEL] entity version not provided, you need to specify deleteAll=true to delete all versions for action $entityName") case None => val fs = - if (results.versions.isEmpty) + if (results.versionMappings.isEmpty) Seq(WhiskAction.get(entityStore, entityName.toDocId) flatMap { entity => WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) }) else - results.versions.map { version => - val id = entityName.copy(version = Some(version)).asString - WhiskAction.get(entityStore, DocId(id)) flatMap { entity => - WhiskAction.del(entityStore, DocInfo ! (id, entity.rev.rev)).map(_ => entity) + results.versionMappings.values.map { docid => + WhiskAction.get(entityStore, docid) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (docid.id, entity.rev.rev)).map(_ => entity) } } val deleteFuture = Future.sequence(fs).andThen { diff --git a/tools/migrate/migrate_to_new_id.py b/tools/migrate/migrate_to_new_id.py index 1392a911d77..06e2a5edc5c 100644 --- a/tools/migrate/migrate_to_new_id.py +++ b/tools/migrate/migrate_to_new_id.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Python script to delete old Activations. +""" /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -26,7 +26,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description="Utility to migrate actions with new ids.") - parser.add_argument("--dbUrl", required=True, help="Server URL of the database, that has to be cleaned of old activations. E.g. 'https://xxx:yyy@domain.couch.com:443'") + parser.add_argument("--dbUrl", required=True, help="Server URL of the database. E.g. 'https://xxx:yyy@domain.couch.com:443'") parser.add_argument("--dbName", required=True, help="Name of the Database of the actions to be migration.") parser.add_argument("--docsPerRequest", type=int, default=200, help="Number of documents handled on each CouchDb Request. Default is 200.") From fc564c9f0cb8b8f38661c85858a1697fd4023b69 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Wed, 12 May 2021 13:39:07 +0800 Subject: [PATCH 24/29] Update some return errors --- .../org/apache/openwhisk/core/controller/Actions.scala | 6 +++--- .../openwhisk/core/controller/test/ActionsApiTests.scala | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index c4c2218ef82..2ffe8b68d83 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -242,7 +242,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with complete(OK, version) }) case Success(_) => - terminate(Forbidden, s"[PUT] entity doesn't has version $version") + terminate(NotFound, s"[PUT] entity doesn't has version $version") case Failure(_) => terminate(InternalServerError) } @@ -551,8 +551,8 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with onComplete(WhiskActionVersionList.get(entityName, entityStore)) { case Success(res) => complete(OK, res) - case Failure(t) => - terminate(Forbidden, forbiddenGetAction(entityName.path.asString)) + case Failure(_) => + terminate(InternalServerError) } //check if execute only is enabled, and if there is a discrepancy between the current user's namespace //and that of the entity we are trying to fetch diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index a54cbf760eb..c0c7a9ec206 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -1616,7 +1616,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Put(s"$collectionPath/${action.name}?defaultVersion=0.0.2", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) - status should be(Forbidden) + status should be(NotFound) } } From 9fad6e3453b6025daa64c69abfa66195e8d2bf53 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Wed, 12 May 2021 15:05:44 +0800 Subject: [PATCH 25/29] Rebase master --- .../v2/test/FunctionPullingContainerPoolTests.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala index e6ed5edc79a..5ddce1ebb8f 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala @@ -105,6 +105,7 @@ class FunctionPullingContainerPoolTests private val exec = CodeExecAsString(RuntimeManifest(actionKind, ImageName("testImage")), "testCode", None) private val memoryLimit = MemoryLimit.STD_MEMORY.toMB.MB private val whiskAction = WhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + private val docId = DocId("actionSpace/actionName@0.0.1") private val invocationNamespace = EntityName("invocationSpace") private val schedulerHost = "127.17.0.1" private val rpcPort = 13001 @@ -114,11 +115,13 @@ class FunctionPullingContainerPoolTests EntityName("bigActionName"), exec, limits = ActionLimits(memory = MemoryLimit(memoryLimit * 2))) + private val bigDocId = DocId("actionSpace/actionName@0.0.1") private val execMetadata = CodeExecMetaDataAsString(exec.manifest, entryPoint = exec.entryPoint) private val actionMetaData = WhiskActionMetaData( whiskAction.namespace, whiskAction.name, + docId, execMetadata, whiskAction.parameters, whiskAction.limits, @@ -129,6 +132,7 @@ class FunctionPullingContainerPoolTests WhiskActionMetaData( bigWhiskAction.namespace, bigWhiskAction.name, + bigDocId, execMetadata, bigWhiskAction.parameters, bigWhiskAction.limits, From 0205d2e319efaa44fd243ca6579dbc4852ba1411 Mon Sep 17 00:00:00 2001 From: "jiang.pengcheng" Date: Thu, 3 Jun 2021 11:00:30 +0800 Subject: [PATCH 26/29] Fix rebase errors --- .../src/main/scala/org/apache/openwhisk/core/WarmUp.scala | 2 ++ .../core/invoker/test/ContainerMessageConsumerTests.scala | 7 ++++++- .../test/ShardingContainerPoolBalancerTests.scala | 1 + .../scheduler/container/test/ContainerManagerTests.scala | 8 ++++++-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala index cd2e205c561..77d59f0ade4 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala @@ -41,6 +41,7 @@ object WarmUp { revision = DocRevision.empty, user = warmUpActionIdentity, activationId = new ActivationIdGenerator {}.make(), + DocId(warmUpAction.asString), rootControllerIndex = controller, blocking = false, content = None, @@ -54,6 +55,7 @@ object WarmUp { val metadata = WhiskActionMetaData( warmUpAction.path, warmUpAction.name, + DocId(warmUpAction.asString), CodeExecMetaDataAsString(manifest, false, entryPoint = None)) ContainerCreationMessage( TransactionId.warmUp, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala index 5ceddfb9384..8234d525ed9 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala @@ -157,6 +157,7 @@ class ContainerMessageConsumerTests sendAckToScheduler(producer)) val exec = CodeExecAsString(RuntimeManifest("nodejs:10", ImageName("testImage")), "testCode", None) + val docId = DocId("testns/testAction@0.0.1") val action = WhiskAction(EntityPath("testns"), EntityName("testAction"), exec, limits = ActionLimits(TimeLimit(1.minute))) put(entityStore, action) @@ -166,6 +167,7 @@ class ContainerMessageConsumerTests WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, @@ -212,6 +214,7 @@ class ContainerMessageConsumerTests sendAckToScheduler(ackConsumer.getProducer())) val exec = CodeExecAsString(RuntimeManifest("nodejs:10", ImageName("testImage")), "testCode", None) + val docId = DocId("testns/testAction2@0.0.1") val whiskAction = WhiskAction(EntityPath("testns"), EntityName("testAction2"), exec, limits = ActionLimits(TimeLimit(1.minute))) val execMetadata = @@ -220,6 +223,7 @@ class ContainerMessageConsumerTests WhiskActionMetaData( whiskAction.namespace, whiskAction.name, + docId, execMetadata, whiskAction.parameters, whiskAction.limits, @@ -294,7 +298,7 @@ class ContainerMessageConsumerTests WarmUp.warmUpAction.name, exec, limits = ActionLimits(TimeLimit(1.minute))) - val doc = put(entityStore, action) + val docId = DocId(action.fullyQualifiedName(true).asString) val execMetadata = CodeExecMetaDataAsString(exec.manifest, entryPoint = exec.entryPoint) @@ -302,6 +306,7 @@ class ContainerMessageConsumerTests WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala index 934cd23f6e8..3ea4902d74d 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala @@ -39,6 +39,7 @@ import scala.concurrent.Future import scala.concurrent.duration._ import org.apache.openwhisk.common.Logging import org.apache.openwhisk.common.NestedSemaphore +import org.apache.openwhisk.common.InvokerHealth import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.WhiskConfig diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala index aa688320138..2260f70c5d0 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala @@ -91,12 +91,14 @@ class ContainerManagerTests val resourcesStrictPolicy = false val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val docId = DocId(s"$testNamespace/$testAction@0.0.1") + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), docId, exec) val execMetadata = CodeExecMetaDataAsString(exec.manifest, entryPoint = exec.entryPoint) val actionMetadata = WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, @@ -776,12 +778,14 @@ class ContainerManagerTests ContainerManager.props(factory(mockJobManager), mockMessaging(), testsid, mockEtcd, config, mockWatcher.ref)) val exec = BlackBoxExec(ExecManifest.ImageName("image"), None, None, native = false, binary = false) - val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val docId = DocId(s"$testNamespace/$testAction@0.0.1") + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), docId, exec) val execMetadata = BlackBoxExecMetaData(exec.image, exec.entryPoint, exec.native, exec.binary) val actionMetadata = WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, From 6a4196808435206b4c0c752d811cebfab5d8c433 Mon Sep 17 00:00:00 2001 From: Brendan Doyle Date: Wed, 15 Feb 2023 11:53:11 -0800 Subject: [PATCH 27/29] Update Actions.scala fix bad conflict resolution --- .../scala/org/apache/openwhisk/core/controller/Actions.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 8922f2b1f52..85f0669dc28 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -249,7 +249,6 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case Failure(_) => entity(as[WhiskActionPut]) { content => val request = content.resolve(user.namespace) - checkLimits <- checkActionLimits(user, content) val check = for { checkLimits <- checkActionLimits(user, content) checkAdditionalPrivileges <- entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap(_ => From 482c05e245d49abeffadf30ca5073dc4c467e87c Mon Sep 17 00:00:00 2001 From: Brendan Doyle Date: Wed, 15 Feb 2023 15:51:37 -0800 Subject: [PATCH 28/29] fix compilation w/ new scheduler code --- .../openwhisk/core/entity/WhiskAction.scala | 2 +- .../openwhisk/core/controller/Actions.scala | 1 - .../core/loadBalancer/FPCPoolBalancer.scala | 1 + .../actions/zippedaction/package-lock.json | 25 ++++++++++++++++++- .../v2/test/ActivationClientProxyTests.scala | 6 +++-- .../FunctionPullingContainerProxyTests.scala | 3 ++- .../test/CreationJobManagerTests.scala | 4 ++- .../queue/test/MemoryQueueTests.scala | 1 + .../queue/test/MemoryQueueTestsFixture.scala | 5 +++- .../queue/test/QueueManagerTests.scala | 8 ++++-- 10 files changed, 46 insertions(+), 10 deletions(-) diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index b22bd52f2d2..aba82be4a0f 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -34,9 +34,9 @@ import org.apache.openwhisk.core.database.{ ArtifactStore, CacheChangeNotification, DocumentFactory, - NoDocumentException, EvictionPolicy, MultipleReadersSingleWriterCache, + NoDocumentException, StaleParameter, WriteTime } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index 85f0669dc28..4954d030db3 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -255,7 +255,6 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with entitlementProvider.check(user, content.exec)) } yield (checkAdditionalPrivileges, checkLimits) - onComplete(check) { case Success(_) => onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala index bf04d1d2a4d..351956fe2bc 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala @@ -537,6 +537,7 @@ class FPCPoolBalancer(config: WhiskConfig, val metadata = ExecutableWhiskActionMetaData( WarmUp.warmUpAction.path, WarmUp.warmUpAction.name, + DocId(WarmUp.warmUpAction.asString), CodeExecMetaDataAsString(manifest, false, entryPoint = None)) CreateQueue( WarmUp.warmUpActionIdentity.namespace.name.asString, diff --git a/tests/dat/actions/zippedaction/package-lock.json b/tests/dat/actions/zippedaction/package-lock.json index 4bbddd42a9c..8ca4ddd42d9 100644 --- a/tests/dat/actions/zippedaction/package-lock.json +++ b/tests/dat/actions/zippedaction/package-lock.json @@ -1,8 +1,31 @@ { "name": "test-action", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "test-action", + "version": "1.0.0", + "license": "Apache 2.0", + "dependencies": { + "prog-quote": "2.0.0" + } + }, + "node_modules/prog-quote": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prog-quote/-/prog-quote-2.0.0.tgz", + "integrity": "sha1-TLBMeosV/zu/kxMQxCsBzSjcMB0=", + "dependencies": { + "random-js": "1.0.8" + } + }, + "node_modules/random-js": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/random-js/-/random-js-1.0.8.tgz", + "integrity": "sha1-lo/WiabyXWwKrHZig94vaIycGQo=" + } + }, "dependencies": { "prog-quote": { "version": "2.0.0", diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala index cf8ef15b2b0..62dc686c496 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala @@ -64,9 +64,10 @@ class ActivationClientProxyTests val timeout = 20.seconds val log = logging - + val testNamespace = "actionSpace" + val testActionName = "actionName" val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testActionName), DocId(s"$testNamespace/$testActionName@0.0.1"), exec) val fqn = action.fullyQualifiedName(true) val rev = action.rev val schedulerHost = "127.17.0.1" @@ -82,6 +83,7 @@ class ActivationClientProxyTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala index 52362256b3d..eb7e43f95d4 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala @@ -125,7 +125,7 @@ class FunctionPullingContainerProxyTests val memoryLimit = 256.MB - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), DocId("actionSpace/actionName@0.0.1"), exec) val fqn = FullyQualifiedEntityName(action.namespace, action.name, Some(action.version)) @@ -135,6 +135,7 @@ class FunctionPullingContainerProxyTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala index 61e8199b7ca..def68ba24d3 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala @@ -68,7 +68,7 @@ class CreationJobManagerTests val schedulerHost = "127.17.0.1" val rpcPort = 13001 val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val execAction = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val execAction = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), DocId(s"$testNamespace/$testAction@0.0.1"), exec) val execMetadata = CodeExecMetaDataAsString(RuntimeManifest(execAction.exec.kind, ImageName("test")), entryPoint = Some("test")) val revision = DocRevision("1-testRev") @@ -76,6 +76,7 @@ class CreationJobManagerTests WhiskActionMetaData( execAction.namespace, execAction.name, + execAction.docId, execMetadata, execAction.parameters, execAction.limits, @@ -342,6 +343,7 @@ class CreationJobManagerTests val actionMetaData = WhiskActionMetaData( execAction.namespace, execAction.name, + execAction.docId, execMetadata, execAction.parameters, execAction.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala index c2c5ae34003..8e31749f30a 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala @@ -1342,6 +1342,7 @@ class MemoryQueueTests WhiskActionMetaData( action.namespace, action.name, + action.docId, execMetadata, action.parameters, action.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index d793c58486f..81a3f63a0a7 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -91,13 +91,14 @@ class MemoryQueueTestsFixture val fqn = FullyQualifiedEntityName(EntityPath(testNamespace), EntityName(testAction), Some(SemVer(0, 0, 1))) val revision = DocRevision("1-testRev") val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), DocId(s"$testNamespace/$testAction@0.0.1"), exec) val execMetadata = CodeExecMetaDataAsString(RuntimeManifest(action.exec.kind, ImageName("test")), entryPoint = Some("test")) val actionMetadata = WhiskActionMetaData( action.namespace, action.name, + action.docId, execMetadata, action.parameters, action.limits, @@ -125,6 +126,7 @@ class MemoryQueueTestsFixture BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -301,6 +303,7 @@ class MemoryQueueTestsFixture BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala index 5baa97ebc82..35f38d99c61 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala @@ -84,7 +84,7 @@ class QueueManagerTests val messageTransId = TransactionId(TransactionId.testing.meta.id) val uuid = UUID() - val action = ExecutableWhiskAction(testEntityPath, testEntityName, testExec) + val action = ExecutableWhiskAction(testEntityPath, testEntityName, DocId(s"${testEntityPath.namespace}/${testEntityName.name}@0.0.1"), testExec) val testLeaderKey = QueueKeys.queue(testInvocationNamespace, action.fullyQualifiedName(false), true) val activationMessage = ActivationMessage( @@ -97,6 +97,7 @@ class QueueManagerTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -143,6 +144,7 @@ class QueueManagerTests WhiskActionMetaData( action.namespace, action.name, + action.docid, exec, action.parameters, action.limits, @@ -458,6 +460,7 @@ class QueueManagerTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -543,6 +546,7 @@ class QueueManagerTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -1108,7 +1112,7 @@ class QueueManagerTests val watcher = TestProbe() val warmUpActionMetaData = - WhiskActionMetaData(warmUpAction.namespace.toPath, warmUpAction.name, testExecMetadata, version = semVer) + WhiskActionMetaData(warmUpAction.namespace.toPath, warmUpAction.name, DocId(s"${warmUpAction.namespace}/${warmUpAction.name}@0.0.1"), testExecMetadata, version = semVer) val warmUpQueueCreationMessage = CreateQueue(warmUpAction.namespace.toString, warmUpAction, testDocRevision, warmUpActionMetaData) From cc569082e94408e606eeb732875ee4630377d199 Mon Sep 17 00:00:00 2001 From: Brendan Doyle Date: Wed, 15 Feb 2023 19:18:51 -0800 Subject: [PATCH 29/29] fix versioned action read on fpcs invoker --- .../openwhisk/core/invoker/ContainerMessageConsumer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala index 416ce50f4d5..593f8a72eaf 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala @@ -69,7 +69,7 @@ class ContainerMessageConsumer( val createContainer = for { identity <- Identity.get(authStore, EntityName(creation.invocationNamespace)) action <- WhiskAction - .get(entityStore, creation.action.toDocId, creation.revision, fromCache = true) + .get(entityStore, creation.whiskActionMetaData.docId, creation.revision, fromCache = true) } yield { // check action limits before creating container action.limits.checkLimits(identity)