📒 Cook Book - Problem 3: Sharding Large Datasets

[benzmuircroft]

Please note that this code is more than likely totally wrong (Until updated/fixed by @5-142857 as discussed on discord)

NIPs

For a full list of the Nostr protocol NIPS (Nostr Implementation Protocols) see: nostr-protocol/nips

📒 Sharding Large Data Sets

P2p applications are great! But, due to there lack of a central sever, users would have to hold all the needed data themselves. In this case it's something that is expected to grow and grow - it's a dictionary! This can grow so large that we will separate words into groups beginning with aa, ab, ac, ect, ba, bb, bc, ect ... So, each user can hold only the parts of the data set that they need and they can drop parts that they don't need at the moment. This is where hyper-nostr comes to the rescue with sharding!

Other assumptions

(Should be clarified)

• This is possible (?)
• If the user is the only one who is keeping a shard, but they want to drop it then ??? happens (?)
• The object from createSwarm holds the publicKey (?)

;(async function(){
  const SDK = await import("hyper-sdk");
  const createSwarm = (await import('hyper-nostr')).default;
  const { generatePrivateKey, getPublicKey, getEventHash, signEvent } = require('nostr-tools');
  const goodbye = require('graceful-goodbye');

  const yourStorageFolder = './dictionary';
  
  const sdk = await SDK.create({
      storage: yourStorageFolder,
      autoJoin: true
  });
  goodbye(_ => sdk.close());
  
  const dictionary = await createSwarm(sdk, 'dictionary');
  const dictionary.priv = generatePrivateKey();
  const dictionary.pub = getPublicKey(dictionary.priv);

  dictionary.sendEvent('REQ', 'ac', {}); // get the shard with only words that start with 'ac'

  await dictionary.update();

  let ev = {
    kind: 1000, // normal (see nip 1)
    created_at: (+new Date() / 1000),
    pubkey: dictionary.pub, 
    word: 'acid',
    definition: 'An acid is a molecule or ion capable of either donating a proton, known as a Brønsted–Lowry acid, or forming a covalent bond with an electron pair, known as a Lewis acid. The first category of acids are the proton donors, or Brønsted–Lowry acids.',
    tags: []
  };

  ev.id = getEventHash(ev);
  ev.sign = signEvent(ev, dictionary.priv);

  dictionary.sendEvent('EVENT', ev); 
  await dictionary.update();

  dictionary.sendEvent('REQ', 'mo', {});  // get the shard with only words that start with 'mo'

  const result = await dictionary.queryEvents({word: 'moose'});

  console.log(result);

  /* // has more than one result
  [
    {
      kind: 1000, // normal (see nip 1)
      created_at: 1234567890000,
      publicKey: 'tvf76dru6r76r7f', 
      word: 'moose',
      definition: 'The moose or elk is the only species in the genus Alces. The moose is the tallest and second-largest land mammal in North America, only falling short of the American buffalo in terms of mass. It is the largest and heaviest extant species of deer.',
      tags: []
    },
    {
      kind: 1000, // normal (see nip 1)
      created_at: 1234567890000,
      publicKey: '675r7vr75eww',
      word: 'moose',
      definition: '(Alces alces), largest member of the deer family Cervidae (order Artiodactyla). Moose are striking in appearance because of their towering size, black colour, long legs, pendulous muzzle, and dangling hairy dewlap (called a bell) and the immense, wide, flat antlers of old bulls.',
      tags: []
    }
  ]
  */
  
  dictionary.sendEvent('CLOSE', 'mo'); // we don't need 'mo' shard at the moment on our computer

})();