18896 Replace STORAGE_BACKEND with STORAGES and support Script running from S3 (#18680)

This commit is contained in:
Arthur Hanson
2025-03-17 07:36:34 -07:00
committed by GitHub
parent ffe035567a
commit 1b4e00aeda
14 changed files with 246 additions and 77 deletions

View File

@@ -1,11 +1,31 @@
import importlib.abc
import importlib.util
import os
from importlib.machinery import SourceFileLoader
import sys
from django.core.files.storage import storages
__all__ = (
'PythonModuleMixin',
)
class CustomStoragesLoader(importlib.abc.Loader):
"""
Custom loader for exec_module to use django-storages instead of the file system.
"""
def __init__(self, filename):
self.filename = filename
def create_module(self, spec):
return None # Use default module creation
def exec_module(self, module):
storage = storages.create_storage(storages.backends["scripts"])
with storage.open(self.filename, 'rb') as f:
code = f.read()
exec(code, module.__dict__)
class PythonModuleMixin:
def get_jobs(self, name):
@@ -33,6 +53,16 @@ class PythonModuleMixin:
return name
def get_module(self):
loader = SourceFileLoader(self.python_name, self.full_path)
module = loader.load_module()
"""
Load the module using importlib, but use a custom loader to use django-storages
instead of the file system.
"""
spec = importlib.util.spec_from_file_location(self.python_name, self.name)
if spec is None:
raise ModuleNotFoundError(f"Could not find module: {self.python_name}")
loader = CustomStoragesLoader(self.name)
module = importlib.util.module_from_spec(spec)
sys.modules[self.python_name] = module
loader.exec_module(module)
return module