Support startFrom on List Blobs

src/blob/handlers/ContainerHandler.ts

@@ -680,7 +680,8 @@ export default class ContainerHandler extends BaseHandler
       options.maxresults,
       marker,
       includeSnapshots,
-      includeUncommittedBlobs
+      includeUncommittedBlobs,
+      request.getQuery("startFrom")
     );
 
     const serviceEndpoint = `${request.getEndpoint()}/${accountName}`;
@@ -786,7 +787,8 @@ export default class ContainerHandler extends BaseHandler
       options.maxresults,
       marker,
       includeSnapshots,
-      includeUncommittedBlobs
+      includeUncommittedBlobs,
+      request.getQuery("startFrom")
     );
 
     const serviceEndpoint = `${request.getEndpoint()}/${accountName}`;

src/blob/persistence/IBlobMetadataStore.ts

@@ -495,7 +495,8 @@ export interface IBlobMetadataStore
     maxResults?: number,
     marker?: string,
     includeSnapshots?: boolean,
-    includeUncommittedBlobs?: boolean
+    includeUncommittedBlobs?: boolean,
+    startFrom?: string
   ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]>;
 
   listAllBlobs(

src/blob/persistence/LokiBlobMetadataStore.ts

@@ -919,7 +919,8 @@ export default class LokiBlobMetadataStore
     maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS,
     marker: string = "",
     includeSnapshots?: boolean,
-    includeUncommittedBlobs?: boolean
+    includeUncommittedBlobs?: boolean,
+    startFrom?: string
   ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]> {
     const query: any = {};
     if (prefix !== "") {
@@ -948,6 +949,12 @@ export default class LokiBlobMetadataStore
         .where((obj) => {
           return obj.name > marker!;
         })
+        .where((obj) => {
+          // startFrom is inclusive where marker is exclusive, and the two
+          // compose: paging a listing that began at startFrom advances the
+          // marker past it anyway.
+          return startFrom === undefined ? true : obj.name >= startFrom;
+        })
         .where((obj) => {
           return includeSnapshots ? true : obj.snapshot.length === 0;
         })

src/blob/persistence/SqlBlobMetadataStore.ts

@@ -1300,7 +1300,8 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
     maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS,
     marker?: string,
     includeSnapshots?: boolean,
-    includeUncommittedBlobs?: boolean
+    includeUncommittedBlobs?: boolean,
+    startFrom?: string
   ): Promise<[BlobModel[], BlobPrefixModel[], any | undefined]> {
     return this.sequelize.transaction(async (t) => {
       await this.assertContainerExists(context, account, container, t);
@@ -1328,6 +1329,19 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
             };
           }
         }
+
+        // startFrom is inclusive where marker is exclusive, and the two
+        // compose: paging a listing that began at startFrom advances the
+        // marker past it anyway.
+        if (startFrom !== undefined) {
+          if (whereQuery.blobName !== undefined) {
+            whereQuery.blobName[Op.gte] = startFrom;
+          } else {
+            whereQuery.blobName = {
+              [Op.gte]: startFrom
+            };
+          }
+        }
       }
       if (!includeSnapshots) {
         whereQuery.snapshot = "";

tests/blob/apis/container.test.ts

@@ -4,6 +4,7 @@ import {
   AccountSASServices,
   AnonymousCredential,
   BlobServiceClient,
+  ContainerClient,
   generateAccountSASQueryParameters,
   newPipeline,
   StorageSharedKeyCredential,
@@ -1712,4 +1713,100 @@ describe("ContainerAPIs", () => {
     assert.strictEqual(false, await containerClient.exists());
     assert.strictEqual(false, await blockBlobClient.exists());
   });
+
+  it("listBlobsFlat with startFrom should begin at that blob name @loki @sql", async () => {
+    // "b&c" needs encoding to survive the query string: a bare & ends the
+    // parameter, leaving a startFrom of "b" that admits one blob too many.
+    const names = ["a", "b", "b&c", "c"];
+    for (const name of names) {
+      await containerClient.getBlockBlobClient(name).upload("", 0);
+    }
+
+    const listFrom = async (startFrom: string) => {
+      const page = (
+        await listBlobsWithStartFrom(startFrom).listBlobsFlat().byPage().next()
+      ).value;
+      return page.segment.blobItems.map((item: any) => item.name);
+    };
+
+    // startFrom is inclusive, unlike the marker
+    assert.deepStrictEqual(await listFrom("b"), ["b", "b&c", "c"]);
+    assert.deepStrictEqual(await listFrom("b&c"), ["b&c", "c"]);
+    // A name no blob carries still starts the listing at the right place
+    assert.deepStrictEqual(await listFrom("b0"), ["c"]);
+    // One before every blob returns them all, as delta lake asks for
+    assert.deepStrictEqual(await listFrom("0"), names);
+
+    for (const name of names) {
+      await containerClient.getBlockBlobClient(name).delete();
+    }
+  });
+
+  it("listBlobsByHierarchy with startFrom should begin at that blob name @loki @sql", async () => {
+    const names = ["a/1", "a/2", "b/1", "c"];
+    for (const name of names) {
+      await containerClient.getBlockBlobClient(name).upload("", 0);
+    }
+
+    const listFrom = async (startFrom: string) => {
+      const page = (
+        await listBlobsWithStartFrom(startFrom)
+          .listBlobsByHierarchy("/")
+          .byPage()
+          .next()
+      ).value;
+      return [
+        (page.segment.blobPrefixes ?? []).map((p: any) => p.name),
+        page.segment.blobItems.map((item: any) => item.name)
+      ];
+    };
+
+    // startFrom selects blobs before the delimiter squashes them into
+    // prefixes, so a prefix survives only while one of its blobs does.
+    assert.deepStrictEqual(await listFrom("0"), [["a/", "b/"], ["c"]]);
+    assert.deepStrictEqual(await listFrom("a/2"), [["a/", "b/"], ["c"]]);
+    assert.deepStrictEqual(await listFrom("b/"), [["b/"], ["c"]]);
+    assert.deepStrictEqual(await listFrom("c"), [[], ["c"]]);
+
+    for (const name of names) {
+      await containerClient.getBlockBlobClient(name).delete();
+    }
+  });
+
+  // The JS SDK has no startFrom yet, so put it on the wire by rewriting the
+  // query of a SAS-authenticated request.  A blob name is arbitrary text, so
+  // it is encoded the way a client would have to encode it.
+  function listBlobsWithStartFrom(startFrom: string): ContainerClient {
+    const storageSharedKeyCredential = new StorageSharedKeyCredential(
+      EMULATOR_ACCOUNT_NAME,
+      EMULATOR_ACCOUNT_KEY
+    );
+    const tmr = new Date();
+    tmr.setDate(tmr.getDate() + 1);
+    const sas = generateAccountSASQueryParameters(
+      {
+        expiresOn: tmr,
+        permissions: AccountSASPermissions.parse("rl"),
+        resourceTypes: AccountSASResourceTypes.parse("sco").toString(),
+        services: AccountSASServices.parse("b").toString(),
+        version: "2020-04-08"
+      },
+      storageSharedKeyCredential as StorageSharedKeyCredential
+    ).toString();
+
+    const pipeline = newPipeline(new AnonymousCredential(), {
+      retryOptions: { maxTries: 1 },
+      keepAliveOptions: { enable: false }
+    });
+    pipeline.factories.unshift(
+      new QueryRequestPolicyFactory(
+        "restype=container",
+        `restype=container&startFrom=${encodeURIComponent(startFrom)}`
+      )
+    );
+    return new BlobServiceClient(
+      `${baseURL}?${sas}`,
+      pipeline
+    ).getContainerClient(containerName);
+  }
 });