diff --git a/syft/pkg/collection.go b/syft/pkg/collection.go
index 6192abfeb..d6956c084 100644
--- a/syft/pkg/collection.go
+++ b/syft/pkg/collection.go
@@ -201,7 +201,11 @@ func (c *Collection) deleteNameFromIndex(p Package) {
 
 	nameIndex := c.idsByName[p.Name]
 	nameIndex.delete(p.id)
-	c.idsByName[p.Name] = nameIndex
+	if len(nameIndex.slice) == 0 {
+		delete(c.idsByName, p.Name)
+	} else {
+		c.idsByName[p.Name] = nameIndex
+	}
 }
 
 func (c *Collection) deleteTypeFromIndex(p Package) {
@@ -209,7 +213,11 @@ func (c *Collection) deleteTypeFromIndex(p Package) {
 
 	typeIndex := c.idsByType[p.Type]
 	typeIndex.delete(p.id)
-	c.idsByType[p.Type] = typeIndex
+	if len(typeIndex.slice) == 0 {
+		delete(c.idsByType, p.Type)
+	} else {
+		c.idsByType[p.Type] = typeIndex
+	}
 }
 
 func (c *Collection) deletePathsFromIndex(p Package) {
diff --git a/syft/pkg/collection_test.go b/syft/pkg/collection_test.go
index f22dc0b48..5fee8d8f6 100644
--- a/syft/pkg/collection_test.go
+++ b/syft/pkg/collection_test.go
@@ -184,6 +184,24 @@ func TestCatalogDeleteRemovesPackages(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "delete idsBy key entries when all deleted",
+			pkgs: []Package{
+				{
+					id:      artifact.ID("pkg:deb/debian/1"),
+					Name:    "debian",
+					Version: "1",
+					Type:    DebPkg,
+					Locations: file.NewLocationSet(
+						file.NewVirtualLocation("/c/path", "/another/path1"),
+					),
+				},
+			},
+			deleteIDs: []artifact.ID{
+				artifact.ID("pkg:deb/debian/1"),
+			},
+			expectedIndexes: expectedIndexes{},
+		},
 	}
 
 	for _, test := range tests {