datachain.yml

- title: Filter a billion samples in seconds
  description:
    Datasets are getting larger, but the ability to iterate rapidly and
    efficiently is as important as ever.
  terminal: |
    from datachain.query import C, DatasetQuery
    from datachain.lib.webdataset import WebDataset
    from datachain.lib.gpt4_vision import DescribeImage

    prompt = "How many people in the image?"

    people = DataChain.from_storage("s3://my-storage/", type="image") \
                      .filter(C.name.glob("*.jpg"))                   \
                      .map(DescribeImage(prompt, max_token = 300))    \
                      .mutate(num_people = int(C.descr))

    people.filter(C.num_people > 3)

- title: Create datasets from queries
  description:
    Save the results of a query in a dataset that you can use to train your ML
    models.
  terminal: |
    from datachain.lib.dc import C, DataChain

    images = DataChain.from_storage("s3://my-storage/", type="image") \
                      .filter(C.name.glob("*.jpg"))

    images.save("fashion-product-images")

- title: Version datasets without copying data
  description:
    No expensive data copies, hash calculations, or data movement. Capture and
    save metadata instead of data.
  terminal: |
    from datachain.lib.dc import C, DataChain

    images = DataChain.from_storage("s3://my-storage/", type="image") \
                      .filter(C.name.glob("*.jpg"))

    images.save("fashion-product-images")