Skip to content

Commit 530c2e9

Browse files
robkooperrobkooper
committed
fix status of spaces
original migration forgot to count files in folders
1 parent 492945b commit 530c2e9

File tree

2 files changed

+88
-81
lines changed

2 files changed

+88
-81
lines changed

‎app/services/mongodb/MongoSalatPlugin.scala‎

Lines changed: 77 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
431431

432432
// Change from User active and serverAdmin flags to single status
433433
updateMongo("change-to-user-status", updateToUserStatus)
434-
434+
435435
// Capture original filename from FRBR metadata supplied by SEAD Migrator
436436
updateMongo("populate-original-filename", updateOriginalFilename)
437437

@@ -450,9 +450,8 @@ class MongoSalatPlugin(app: Application) extends Plugin {
450450
// Updates extractors enabled and disabled in a space
451451
updateMongo("update-space-extractors-selection", updateSpaceExtractorsSelection)
452452

453-
// Adds space bytes to space
454-
updateMongo(updateKey = "update-space-bytes", updateSpaceBytes)
455-
updateMongo(updateKey = "update-space-files", updateSpaceFiles)
453+
// Adds status information to space
454+
updateMongo(updateKey = "update-space-status", updateSpaceStatus)
456455
}
457456

458457
private def updateMongo(updateKey: String, block: () => Unit): Unit = {
@@ -478,14 +477,14 @@ class MongoSalatPlugin(app: Application) extends Plugin {
478477

479478
private def addDateMovedToTrashCollections() {
480479
val q = MongoDBObject()
481-
val s = MongoDBObject("$set" -> MongoDBObject("dateMovedToTrash" -> None, "trash"->false))
482-
collection("collections").update(q,s, multi=true)
480+
val s = MongoDBObject("$set" -> MongoDBObject("dateMovedToTrash" -> None, "trash" -> false))
481+
collection("collections").update(q, s, multi = true)
483482
}
484483

485484
private def addDateMovedToTrashDatasets() {
486485
val q = MongoDBObject()
487-
val s = MongoDBObject("$set" -> MongoDBObject("dateMovedToTrash" -> None, "trash"->false))
488-
collection("datasets").update(q,s, multi=true)
486+
val s = MongoDBObject("$set" -> MongoDBObject("dateMovedToTrash" -> None, "trash" -> false))
487+
collection("datasets").update(q, s, multi = true)
489488
}
490489

491490
private def updateMongoChangeUserType() {
@@ -1223,8 +1222,8 @@ class MongoSalatPlugin(app: Application) extends Plugin {
12231222
var lastId: ObjectId = null
12241223
var lastCollection: String = null
12251224
var count = 0
1226-
collection("metadata").find().sort(MongoDBObject("attachedTo" -> 1)).foreach{d =>
1227-
d.getAs[DBObject]("attachedTo").foreach{at =>
1225+
collection("metadata").find().sort(MongoDBObject("attachedTo" -> 1)).foreach { d =>
1226+
d.getAs[DBObject]("attachedTo").foreach { at =>
12281227
(at.getAs[ObjectId]("_id"), at.getAs[String]("resourceType")) match {
12291228
case (Some(id), Some(coll)) => {
12301229
if (id != lastId) {
@@ -1311,29 +1310,29 @@ class MongoSalatPlugin(app: Application) extends Plugin {
13111310
userpasses.foreach { user =>
13121311
(user.getAs[ObjectId]("_id"), user.getAs[String]("email"),
13131312
user.getAsOrElse[DBObject]("identityId", new MongoDBObject()).getAs[String]("userId")) match {
1314-
case (Some(userId), Some(email), Some(username)) => {
1315-
try {
1316-
// Find if user exists with lowercase email already
1317-
val conflicts = collection("social.users").count(MongoDBObject(
1318-
"_id" -> MongoDBObject("$ne" -> userId),
1319-
"identityId" -> MongoDBObject("userId" -> username.toLowerCase, "providerId" -> "userpass")))
1320-
1321-
if (conflicts == 0) {
1322-
collection("social.users").update(MongoDBObject("_id" -> userId),
1323-
MongoDBObject("$set" -> MongoDBObject(
1324-
"email" -> email.toLowerCase,
1325-
"identityId" -> MongoDBObject("userId" -> username.toLowerCase, "providerId" -> "userpass"))), upsert = false, multi = true)
1326-
} else {
1327-
// If there's already an account with lowercase email, deactivate this account
1328-
collection("social.users").update(MongoDBObject("_id" -> userId),
1329-
MongoDBObject("$set" -> MongoDBObject("active" -> false)), upsert = false, multi = true)
1330-
}
1331-
} catch {
1332-
case e: BSONException => Logger.error("Unable to update email for user with id: " + user)
1313+
case (Some(userId), Some(email), Some(username)) => {
1314+
try {
1315+
// Find if user exists with lowercase email already
1316+
val conflicts = collection("social.users").count(MongoDBObject(
1317+
"_id" -> MongoDBObject("$ne" -> userId),
1318+
"identityId" -> MongoDBObject("userId" -> username.toLowerCase, "providerId" -> "userpass")))
1319+
1320+
if (conflicts == 0) {
1321+
collection("social.users").update(MongoDBObject("_id" -> userId),
1322+
MongoDBObject("$set" -> MongoDBObject(
1323+
"email" -> email.toLowerCase,
1324+
"identityId" -> MongoDBObject("userId" -> username.toLowerCase, "providerId" -> "userpass"))), upsert = false, multi = true)
1325+
} else {
1326+
// If there's already an account with lowercase email, deactivate this account
1327+
collection("social.users").update(MongoDBObject("_id" -> userId),
1328+
MongoDBObject("$set" -> MongoDBObject("active" -> false)), upsert = false, multi = true)
13331329
}
1330+
} catch {
1331+
case e: BSONException => Logger.error("Unable to update email for user with id: " + user)
13341332
}
1335-
case _ => Logger.error("Missing user fields when updating email case")
13361333
}
1334+
case _ => Logger.error("Missing user fields when updating email case")
1335+
}
13371336
}
13381337
}
13391338

@@ -1455,21 +1454,21 @@ class MongoSalatPlugin(app: Application) extends Plugin {
14551454
}
14561455

14571456
private def updateAvatarUrl() {
1458-
val q = MongoDBObject("avatarUrl" -> "^http://www.gravatar.com".r)
1457+
val q = MongoDBObject("avatarUrl" -> "^http://www.gravatar.com".r)
14591458
collection("social.users").find(q).foreach { user =>
14601459
val avatar_url = user.getAsOrElse[String]("avatarUrl", "")
1461-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1460+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
14621461
val index = avatar_url.lastIndexOf("/")
14631462
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
14641463
user.put("avatarUrl", new_gravatar)
14651464
}
14661465
collection("social.users").save(user, WriteConcern.Safe)
14671466
}
1468-
collection("events").find(MongoDBObject("user.avatarURL" -> "^http://www.gravatar.com".r)).foreach{ event =>
1467+
collection("events").find(MongoDBObject("user.avatarURL" -> "^http://www.gravatar.com".r)).foreach { event =>
14691468
event.getAs[DBObject]("user") match {
14701469
case Some(mini_user) => {
14711470
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1472-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1471+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
14731472
val index = avatar_url.lastIndexOf("/")
14741473
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
14751474
mini_user.put("avatarURL", new_gravatar)
@@ -1480,7 +1479,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
14801479
event.getAs[DBObject]("targetuser") match {
14811480
case Some(mini_user) => {
14821481
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1483-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1482+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
14841483
val index = avatar_url.lastIndexOf("/")
14851484
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
14861485
mini_user.put("avatarURL", new_gravatar)
@@ -1490,11 +1489,11 @@ class MongoSalatPlugin(app: Application) extends Plugin {
14901489
}
14911490
collection("events").save(event, WriteConcern.Safe)
14921491
}
1493-
collection("collections").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)).foreach{ c =>
1492+
collection("collections").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)).foreach { c =>
14941493
c.getAs[DBObject]("author") match {
14951494
case Some(mini_user) => {
14961495
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1497-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1496+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
14981497
val index = avatar_url.lastIndexOf("/")
14991498
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15001499
mini_user.put("avatarURL", new_gravatar)
@@ -1504,11 +1503,11 @@ class MongoSalatPlugin(app: Application) extends Plugin {
15041503
case _ => Logger.info("No miniuser associated with the collection ")
15051504
}
15061505
}
1507-
collection("datasets").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)).foreach{ dataset =>
1506+
collection("datasets").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)).foreach { dataset =>
15081507
dataset.getAs[DBObject]("author") match {
15091508
case Some(mini_user) => {
15101509
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1511-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1510+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
15121511
val index = avatar_url.lastIndexOf("/")
15131512
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15141513
mini_user.put("avatarURL", new_gravatar)
@@ -1519,11 +1518,11 @@ class MongoSalatPlugin(app: Application) extends Plugin {
15191518
case _ => Logger.info("No miniuser associated with the dataset ")
15201519
}
15211520
}
1522-
collection("folders").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r))foreach{ folder =>
1521+
collection("folders").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)) foreach { folder =>
15231522
folder.getAs[DBObject]("author") match {
15241523
case Some(mini_user) => {
15251524
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1526-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1525+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
15271526
val index = avatar_url.lastIndexOf("/")
15281527
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15291528
mini_user.put("avatarURL", new_gravatar)
@@ -1534,11 +1533,11 @@ class MongoSalatPlugin(app: Application) extends Plugin {
15341533
case _ => Logger.info("No miniuser associated with the folder")
15351534
}
15361535
}
1537-
collection("uploads").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r))foreach{ file =>
1536+
collection("uploads").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)) foreach { file =>
15381537
file.getAs[DBObject]("author") match {
15391538
case Some(mini_user) => {
15401539
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1541-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1540+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
15421541
val index = avatar_url.lastIndexOf("/")
15431542
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15441543
mini_user.put("avatarURL", new_gravatar)
@@ -1549,11 +1548,11 @@ class MongoSalatPlugin(app: Application) extends Plugin {
15491548
case _ => Logger.info("No miniuser associated with the file")
15501549
}
15511550
}
1552-
collection("comments").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r))foreach{ comment =>
1551+
collection("comments").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)) foreach { comment =>
15531552
comment.getAs[DBObject]("author") match {
15541553
case Some(mini_user) => {
15551554
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1556-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1555+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
15571556
val index = avatar_url.lastIndexOf("/")
15581557
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15591558
mini_user.put("avatarURL", new_gravatar)
@@ -1564,11 +1563,11 @@ class MongoSalatPlugin(app: Application) extends Plugin {
15641563
case _ => Logger.info("No miniuser associated with the comment ")
15651564
}
15661565
}
1567-
collection("curationObjects").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r))foreach{ event =>
1566+
collection("curationObjects").find(MongoDBObject("author.avatarURL" -> "^http://www.gravatar.com".r)) foreach { event =>
15681567
event.getAs[DBObject]("author") match {
15691568
case Some(mini_user) => {
15701569
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1571-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1570+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
15721571
val index = avatar_url.lastIndexOf("/")
15731572
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15741573
mini_user.put("avatarURL", new_gravatar)
@@ -1579,26 +1578,26 @@ class MongoSalatPlugin(app: Application) extends Plugin {
15791578
case _ => Logger.info("No miniuser associated with the curation Object ")
15801579
}
15811580
}
1582-
collection("metadata").find(MongoDBObject("creator.typeOfAgent" -> "cat:user", "creator.user.avatarURL" -> "^http://www.gravatar.com".r) )foreach{ metadata =>
1581+
collection("metadata").find(MongoDBObject("creator.typeOfAgent" -> "cat:user", "creator.user.avatarURL" -> "^http://www.gravatar.com".r)) foreach { metadata =>
15831582
metadata.getAs[DBObject]("creator") match {
15841583
case Some(creator) => {
15851584
val typeOfAgent = creator.getAsOrElse("typeOfAgent", "")
1586-
if(typeOfAgent == "cat:user") {
1585+
if (typeOfAgent == "cat:user") {
15871586
creator.getAs[DBObject]("user") match {
15881587
case Some(mini_user) => {
15891588
val avatar_url = mini_user.getAsOrElse("avatarURL", "")
1590-
if(avatar_url.indexOf("http://www.gravatar.com") == 0 ) {
1589+
if (avatar_url.indexOf("http://www.gravatar.com") == 0) {
15911590
val index = avatar_url.lastIndexOf("/")
15921591
val new_gravatar = "https://www.gravatar.com/avatar" + avatar_url.substring(index)
15931592
mini_user.put("avatarURL", new_gravatar)
15941593

15951594
}
15961595
collection("metadata").save(metadata, WriteConcern.Safe)
1597-
}
1596+
}
15981597
case _ => Logger.info("No miniuser associated with the curation Object ")
1598+
}
15991599
}
16001600
}
1601-
}
16021601
case _ => Logger.info("No agent associated with the curation Object ")
16031602
}
16041603
}
@@ -1620,13 +1619,13 @@ class MongoSalatPlugin(app: Application) extends Plugin {
16201619
val path = content.getAsOrElse[String]("Upload Path", "")
16211620
if (path.length > 0) {
16221621
if (path.lastIndexOf("/") >= 0) {
1623-
Logger.info("Assigning name/: " + path.substring(path.lastIndexOf("/")+1) + " from path " + path)
1622+
Logger.info("Assigning name/: " + path.substring(path.lastIndexOf("/") + 1) + " from path " + path)
16241623
md.getAs[DBObject]("attachedTo") match {
16251624
case Some(ref) => {
1626-
collection("uploads").update(MongoDBObject("_id" -> new ObjectId(ref.get("_id").toString())),
1627-
MongoDBObject("$set" -> MongoDBObject(
1628-
"originalname" -> path.substring(path.lastIndexOf("/")+1))), false, false, WriteConcern.Safe)
1629-
1625+
collection("uploads").update(MongoDBObject("_id" -> new ObjectId(ref.get("_id").toString())),
1626+
MongoDBObject("$set" -> MongoDBObject(
1627+
"originalname" -> path.substring(path.lastIndexOf("/") + 1))), false, false, WriteConcern.Safe)
1628+
16301629
}
16311630
case _ => Logger.info("Nope")
16321631
}
@@ -1659,16 +1658,16 @@ class MongoSalatPlugin(app: Application) extends Plugin {
16591658

16601659
private def updateEditorRole(): Unit = {
16611660
val query = MongoDBObject("name" -> "Editor")
1662-
collection("roles").find(query).foreach {role =>
1661+
collection("roles").find(query).foreach { role =>
16631662
role.put("permissions", Permission.EDITOR_PERMISSIONS.map(_.toString).toSet)
16641663
collection("roles").save(role, WriteConcern.Safe)
16651664
}
1666-
collection("social.users").foreach{user =>
1665+
collection("social.users").foreach { user =>
16671666
val userSpaceRoles = user.getAsOrElse[MongoDBList]("spaceandrole", MongoDBList.empty)
1668-
userSpaceRoles.foreach{ userSpaceRole =>
1667+
userSpaceRoles.foreach { userSpaceRole =>
16691668
val tempUserSpace = userSpaceRole.asInstanceOf[BasicDBObject]
16701669
val tempRole = tempUserSpace.get("role").asInstanceOf[BasicDBObject]
1671-
if(tempRole.get("name") == "Editor") {
1670+
if (tempRole.get("name") == "Editor") {
16721671
tempRole.put("permissions", Permission.EDITOR_PERMISSIONS.map(_.toString).toSet)
16731672
}
16741673
}
@@ -1692,33 +1691,34 @@ class MongoSalatPlugin(app: Application) extends Plugin {
16921691
print("DONE")
16931692
}
16941693

1695-
private def updateSpaceBytes(): Unit = {
1696-
val spaces = collection("spaces.projects").find().toList.foreach{ space =>
1694+
private def updateSpaceStatus(): Unit = {
1695+
collection("spaces.projects").find().toList.foreach { space =>
16971696
var currentSpaceBytes: Long = 0
1697+
var fileCount: Integer = 0
16981698
val spaceId = space.get("_id")
16991699
val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList
1700-
spaceDatasets.foreach{ spaceDataset =>
1700+
spaceDatasets.foreach { spaceDataset =>
17011701
val datasetFileIds = spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty)
1702-
datasetFileIds.foreach{ fileId =>
1702+
fileCount += datasetFileIds.length
1703+
datasetFileIds.foreach { fileId =>
17031704
collection("uploads").findOne(MongoDBObject("_id" -> fileId)) match {
17041705
case Some(file) => currentSpaceBytes += file.get("length").asInstanceOf[Long]
17051706
case None => Logger.info(s"Could not find file ${fileId} in space ${spaceId}")
17061707
}
1708+
val folders = collection("folders").find(MongoDBObject("parentDatasetId" -> spaceDataset.get("_id"))).toList
1709+
folders.foreach { folder =>
1710+
val folderFileIds = folder.getAsOrElse[MongoDBList]("files", MongoDBList.empty)
1711+
fileCount += folderFileIds.length
1712+
folderFileIds.foreach { fileId =>
1713+
collection("uploads").findOne(MongoDBObject("_id" -> fileId)) match {
1714+
case Some(file) => currentSpaceBytes += file.get("length").asInstanceOf[Long]
1715+
case None => Logger.info(s"Could not find file ${fileId} in space ${spaceId}")
1716+
}
1717+
}
1718+
}
17071719
}
17081720
}
17091721
collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes))
17101722
}
17111723
}
1712-
1713-
private def updateSpaceFiles(): Unit = {
1714-
collection("spaces.projects").find().toList.foreach{ space =>
1715-
var fileCount: Integer = 0
1716-
val spaceId = space.get("_id")
1717-
val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList
1718-
spaceDatasets.foreach{ spaceDataset =>
1719-
fileCount += spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty).length
1720-
}
1721-
collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("fileCount" -> fileCount))
1722-
}
1723-
}
17241724
}

0 commit comments

Comments
 (0)