Revert "[flutter_tools] cache more directories (#112651)"

This reverts commit 99fb2d36e80737c16b51b3e014fc996c901fdabe.
diff --git a/packages/flutter_tools/lib/src/asset.dart b/packages/flutter_tools/lib/src/asset.dart
index 61bee0e..17b728c 100644
--- a/packages/flutter_tools/lib/src/asset.dart
+++ b/packages/flutter_tools/lib/src/asset.dart
@@ -1019,7 +1019,6 @@
 
   final FileSystem _fileSystem;
   final Map<String, List<String>> _cache = <String, List<String>>{};
-  final Map<String, List<File>> _variantsPerFolder = <String, List<File>>{};
 
   List<String> variantsFor(String assetPath) {
     final String directory = _fileSystem.path.dirname(assetPath);
@@ -1031,26 +1030,25 @@
     if (_cache.containsKey(assetPath)) {
       return _cache[assetPath]!;
     }
-    if (!_variantsPerFolder.containsKey(directory)) {
-      _variantsPerFolder[directory] = _fileSystem.directory(directory)
-        .listSync()
-        .whereType<Directory>()
-        .where((Directory dir) => _assetVariantDirectoryRegExp.hasMatch(dir.basename))
-        .expand((Directory dir) => dir.listSync())
-        .whereType<File>()
-        .toList();
-    }
+
+    final List<FileSystemEntity> entitiesInDirectory = _fileSystem.directory(directory).listSync();
+
     final File assetFile = _fileSystem.file(assetPath);
-    final List<File> potentialVariants = _variantsPerFolder[directory]!;
-    final String basename = assetFile.basename;
-    return _cache[assetPath] = <String>[
+    final List<String> pathsOfVariants = <String>[
       // It's possible that the user specifies only explicit variants (e.g. .../1x/asset.png),
       // so there does not necessarily need to be a file at the given path.
       if (assetFile.existsSync())
         assetPath,
-      ...potentialVariants
-        .where((File file) => file.basename == basename)
+      ...entitiesInDirectory
+        .whereType<Directory>()
+        .where((Directory dir) => _assetVariantDirectoryRegExp.hasMatch(dir.basename))
+        .expand((Directory dir) => dir.listSync())
+        .whereType<File>()
+        .where((File file) => file.basename == assetFile.basename)
         .map((File file) => file.path),
     ];
+
+    _cache[assetPath] = pathsOfVariants;
+    return pathsOfVariants;
   }
 }
diff --git a/packages/flutter_tools/test/general.shard/asset_bundle_test.dart b/packages/flutter_tools/test/general.shard/asset_bundle_test.dart
index ff4e3dc..e2cd837 100644
--- a/packages/flutter_tools/test/general.shard/asset_bundle_test.dart
+++ b/packages/flutter_tools/test/general.shard/asset_bundle_test.dart
@@ -4,6 +4,7 @@
 
 import 'dart:convert';
 
+import 'package:file/file.dart';
 import 'package:file/memory.dart';
 import 'package:flutter_tools/src/artifacts.dart';
 import 'package:flutter_tools/src/asset.dart';