From 2a9e0db6cdd937bde992f18d1e432e22648b6e8a Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 15 Jul 2026 11:50:35 +0200 Subject: [PATCH 01/17] refactor: Replace StoredProcedure with direct sql calls in FileManagerPS --- .../FileManager/FileManagerPs.py | 215 +++++++++++------- .../DB/FileCatalogWithFkAndPsDB.sql | 17 ++ 2 files changed, 144 insertions(+), 88 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index b9204bb6449..bc166fdf78f 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -4,7 +4,7 @@ from DIRAC import S_OK, S_ERROR from DIRAC.DataManagementSystem.DB.FileCatalogComponents.FileManager.FileManagerBase import FileManagerBase -from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks +from DIRAC.Core.Utilities.List import intListToString, breakListIntoChunks from DIRAC.Core.Utilities.TimeUtilities import DiracTime # The logic of some methods is basically a copy/paste from the FileManager class, @@ -67,13 +67,17 @@ def _findFileIDs(self, lfns, connection=False): if len(lfns) == 1: lfn = list(lfns)[0] # if lfns is a dict, list(lfns) returns lfns.keys() pathPart, filePart = os.path.split(lfn) - result = self.db.executeStoredProcedure( - "ps_get_file_id_from_lfn", (pathPart, filePart, "ret1"), outputIds=[2] + req = ( + "SELECT f.FileID " + "FROM FC_Files f " + "JOIN FC_DirectoryList d ON d.DirID = f.DirID " + "WHERE d.Name = %s AND f.FileName = %s" ) + result = self.db._query(req, args=(pathPart, filePart), conn=connection) if not result["OK"]: return result - fileId = result["Value"][0] + fileId = result["Value"][0][0] if result["Value"] else 0 if not fileId: failed[lfn] = "No such file or directory" @@ -95,11 +99,10 @@ def _findFileIDs(self, lfns, connection=False): fileNames = filesInDirDict[dirPath] dirID = directoryPathToIds[dirPath] - formatedFileNames = stringListToString(fileNames) - - result = self.db.executeStoredProcedureWithCursor( - "ps_get_file_ids_from_dir_id", (dirID, formatedFileNames) - ) + req = "SELECT FileID, FileName FROM FC_Files WHERE DirID = %s AND FileName IN (" + req += ",".join(["%s"] * len(fileNames)) + req += ")" + result = self.db._query(req, args=(dirID, *fileNames), conn=connection) if not result["OK"]: return result for fileID, fileName in result["Value"]: @@ -135,14 +138,26 @@ def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, if "FileID" not in metadata: metadata.append("FileID") - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileNames = stringListToString(fileNames) - fStatus = stringListToString(self.db.visibleFileStatus) + req = """SELECT FileName, DirID, f.FileID, Size, f.uid, UserName, f.gid, GroupName, s.Status, + GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode + FROM FC_Files f + JOIN FC_Users u ON f.UID = u.UID + JOIN FC_Groups g ON f.GID = g.GID + JOIN FC_Statuses s ON f.Status = s.StatusID + WHERE DirID = %s""" + args = [dirID] + if not allStatus: + req += " AND s.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleFileStatus)) + req += ")" + args.extend(self.db.visibleFileStatus) + if fileNames: + req += " AND f.FileName IN (" + req += ",".join(["%s"] * len(fileNames)) + req += ")" + args.extend(fileNames) - specificFiles = True if len(fileNames) else False - result = self.db.executeStoredProcedureWithCursor( - "ps_get_all_info_for_files_in_dir", (dirID, specificFiles, formatedFileNames, allStatus, fStatus) - ) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -187,9 +202,13 @@ def _getFileMetadataByID(self, fileIDs, connection=False): ["FileID", "Size", "UID", "GID", "s.Status", "GUID", "CreationDate"] """ - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileIds = intListToString(fileIDs) - result = self.db.executeStoredProcedureWithCursor("ps_get_all_info_for_file_ids", (formatedFileIds,)) + req = """SELECT f.FileID, Size, UID, GID, s.Status, GUID, CreationDate + FROM FC_Files f + JOIN FC_Statuses s ON f.Status = s.StatusID + WHERE f.FileID IN (""" + req += ",".join(["%s"] * len(fileIDs)) + req += ")" + result = self.db._query(req, args=fileIDs, conn=connection) if not result["OK"]: return result @@ -346,9 +365,10 @@ def _getFileIDFromGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] - # formatedGuids = ','.join( [ '"%s"' % guid for guid in guids ] ) - formatedGuids = stringListToString(guids) - result = self.db.executeStoredProcedureWithCursor("ps_get_file_ids_from_guids", (formatedGuids,)) + req = "SELECT GUID, FileID FROM FC_Files WHERE GUID IN (" + req += ",".join(["%s"] * len(guids)) + req += ")" + result = self.db._query(req, args=guids, conn=connection) if not result["OK"]: return result @@ -366,8 +386,15 @@ def getLFNForGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] - formatedGuids = stringListToString(guids) - result = self.db.executeStoredProcedureWithCursor("ps_get_lfns_from_guids", (formatedGuids,)) + req = ( + 'SELECT GUID, CONCAT(d.Name, "/", f.FileName) ' + "FROM FC_Files f " + "JOIN FC_DirectoryList d on f.DirID = d.DirID " + "WHERE GUID IN (" + ) + req += ",".join(["%s"] * len(guids)) + req += ")" + result = self.db._query(req, args=guids, conn=connection) if not result["OK"]: return result @@ -578,16 +605,19 @@ def _getRepIDsForReplica(self, replicaTuples, connection=False): replicaDict = {} - for fileID, seID in replicaTuples: - result = self.db.executeStoredProcedure("ps_get_replica_id", (fileID, seID, "repIdOut"), outputIds=[2]) - if not result["OK"]: - return result + if not replicaTuples: + return S_OK(replicaDict) - repID = result["Value"][0] + req = "SELECT RepID, FileID, SEID FROM FC_Replicas WHERE (FileID, SEID) IN (" + req += ",".join(["(%s,%s)"] * len(replicaTuples)) + req += ")" + args = tuple(value for replicaTuple in replicaTuples for value in replicaTuple) + result = self.db._query(req, args=args, conn=connection) + if not result["OK"]: + return result - # if the replica exists, we add it to the dict - if repID: - replicaDict.setdefault(fileID, {}).setdefault(seID, repID) + for repID, fileID, seID in result["Value"]: + replicaDict.setdefault(fileID, {}).setdefault(seID, repID) return S_OK(replicaDict) @@ -674,11 +704,12 @@ def _setReplicaStatus(self, fileID, se, status, connection=False): return res seID = res["Value"] - result = self.db.executeStoredProcedureWithCursor("ps_set_replica_status", (fileID, seID, statusID)) + req = "UPDATE FC_Replicas SET Status = %s WHERE FileID = %s AND SEID = %s" + result = self.db._update(req, args=(statusID, fileID, seID), conn=connection) if not result["OK"]: return result - affected = result["Value"][0][0] # Affected is the number of raws updated + affected = result["Value"] if not affected: return S_ERROR("Replica does not exist") @@ -734,36 +765,18 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): """ connection = self._getConnection(connection) - # The PS associated with a given parameter - psNames = { - "UID": "ps_set_file_uid", - "GID": "ps_set_file_gid", - "Status": "ps_set_file_status", - "Mode": "ps_set_file_mode", - } - - psName = psNames.get(paramName, None) - - # If there is an associated procedure, we go for it - if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (fileID, paramValue)) - if not result["OK"]: - return result - - _affected = result["Value"][0][0] - # If affected = 0, the file does not exist, but who cares... - - # In case this is a 'new' parameter, we have a failback solution, but we - # should add a specific ps for it - else: - if not self.db._checkIdentifier(paramName)["OK"]: - return S_ERROR(f"ParamName is invalid: {paramName}") - req = "UPDATE FC_Files SET " - req += f"{paramName}=%s, ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (" - req += ",".join(["%s"] * len(fileID)) - req += ")" - args = [paramValue] + fileID - return self.db._update(req, args=args, conn=connection) + if not isinstance(fileID, (list, tuple)): + fileID = [fileID] + if not self.db._checkIdentifier(paramName)["OK"]: + return S_ERROR(f"ParamName is invalid: {paramName}") + req = "UPDATE FC_Files SET " + req += f"{paramName}=%s, ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (" + req += ",".join(["%s"] * len(fileID)) + req += ")" + args = [paramValue] + fileID + result = self.db._update(req, args=args, conn=connection) + if not result["OK"]: + return result return S_OK() @@ -796,17 +809,23 @@ def _getFileReplicas(self, fileIDs, fields_input=None, allStatus=False, connecti # non existing replicas replicas = {fileID: {} for fileID in fileIDs} - # Format the status to be used in a IN clause in the stored procedure - fStatus = stringListToString(self.db.visibleReplicaStatus) - fieldNames = ["FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"] for chunks in breakListIntoChunks(fileIDs, 1000): - # Format the FileIDs to be used in a IN clause in the stored procedure - formatedFileIds = intListToString(chunks) - result = self.db.executeStoredProcedureWithCursor( - "ps_get_all_info_of_replicas_bulk", (formatedFileIds, allStatus, fStatus) - ) + req = """SELECT FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN + FROM FC_Replicas r + JOIN FC_StorageElements se on r.SEID = se.SEID + JOIN FC_Statuses st on r.Status = st.StatusID + WHERE FileID IN (""" + req += ",".join(["%s"] * len(chunks)) + req += ")" + args = list(chunks) + if not allStatus: + req += " AND st.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleReplicaStatus)) + req += ")" + args.extend(self.db.visibleReplicaStatus) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -829,11 +848,12 @@ def countFilesInDir(self, dirId): :returns: S_OK(value) or S_ERROR """ - result = self.db.executeStoredProcedure("ps_count_files_in_dir", (dirId, "ret1"), outputIds=[1]) + req = "SELECT count(FileID) FROM FC_Files WHERE DirID = %s" + result = self.db._query(req, args=(dirId,)) if not result["OK"]: return result - res = S_OK(result["Value"][0]) + res = S_OK(result["Value"][0][0] if result["Value"] else 0) return res ########################################################################## @@ -880,16 +900,24 @@ def getDirectoryReplicas(self, dirID, path, allStatus=False, connection=False): values from the configuration """ - # We format the visible file/replica satus so we can give it as argument to the ps - # It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'-- - # fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] ) - # rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] ) - fStatus = stringListToString(self.db.visibleFileStatus) - rStatus = stringListToString(self.db.visibleReplicaStatus) + req = """SELECT f.FileName, f.FileID, s.SEName, r.PFN + FROM FC_Replicas r + JOIN FC_Files f on f.FileID = r.FileID + JOIN FC_StorageElements s on s.SEID = r.SEID + JOIN FC_Statuses fst on f.Status = fst.StatusID + JOIN FC_Statuses rst on r.Status = rst.StatusID + WHERE DirID = %s""" + args = [dirID] + if not allStatus: + req += " AND fst.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleFileStatus)) + req += ") AND rst.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleReplicaStatus)) + req += ")" + args.extend(self.db.visibleFileStatus) + args.extend(self.db.visibleReplicaStatus) - result = self.db.executeStoredProcedureWithCursor( - "ps_get_replicas_for_files_in_dir", (dirID, allStatus, fStatus, rStatus) - ) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -906,9 +934,13 @@ def _getFileLFNs(self, fileIDs): successful = {} for chunks in breakListIntoChunks(fileIDs, 1000): - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileIds = intListToString(chunks) - result = self.db.executeStoredProcedureWithCursor("ps_get_full_lfn_for_file_ids", (formatedFileIds,)) + req = """SELECT f.FileID, CONCAT(d.Name, "/", f.FileName) + FROM FC_Files f + JOIN FC_DirectoryList d ON f.DirID = d.DirID + WHERE f.FileID IN (""" + req += ",".join(["%s"] * len(chunks)) + req += ")" + result = self.db._query(req, args=chunks) if not result["OK"]: return result @@ -938,6 +970,13 @@ def getSEDump(self, seNames): return res seIDs.append(res["Value"]) - formatedSEIds = intListToString(seIDs) + req = """SELECT SEName, CONCAT(d.Name, "/", f.FileName), f.Checksum, f.Size + FROM FC_Files f + JOIN FC_Replicas r on f.FileID = r.FileID + JOIN FC_DirectoryList d on d.DirID = f.DirID + JOIN FC_StorageElements s on r.SEID = s.SEID + WHERE s.SEID IN (""" + req += ",".join(["%s"] * len(seIDs)) + req += ")" - return self.db.executeStoredProcedureWithCursor("ps_get_se_dump", (formatedSEIds,)) + return self.db._query(req, args=seIDs) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql index 0c00503540d..e0524bab338 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql @@ -565,6 +565,7 @@ DELIMITER ; -- ps_count_files_in_dir : counts how many files are in a given directory -- dir_id : directory id -- countFile (out value): amount of files +-- DEPRECATED: FileManagerPs.countFilesInDir now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_count_files_in_dir; DELIMITER // @@ -634,6 +635,7 @@ DELIMITER ; -- visibleFileStatus : status of files to be considered -- visibleReplicaStatus : status of replicas to be considered -- outputs : FileName, FileID, SEName, PFN +-- DEPRECATED: FileManagerPs.getDirectoryReplicas now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_replicas_for_files_in_dir; DELIMITER // @@ -661,6 +663,7 @@ DELIMITER ; -- dirName : name of the directory -- fileName : name of the file -- file_id (out) : id of the file +-- DEPRECATED: FileManagerPs._findFileIDs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_id_from_lfn; DELIMITER // @@ -683,6 +686,7 @@ DELIMITER ; -- dir_id : directory id -- file_names : names of the files we are interested in -- output : FileID, FileName +-- DEPRECATED: FileManagerPs._findFileIDs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_ids_from_dir_id; DELIMITER // @@ -710,6 +714,7 @@ DELIMITER ; -- visibleFileStatus : list of status we are interested in -- output : FileName, DirID, f.FileID, Size, f.uid, UserName, f.gid, GroupName, s.Status, -- GUID, Checksum, ChecksumType, Type, CreationDate,ModificationDate, Mode +-- DEPRECATED: FileManagerPs._getDirectoryFiles now uses a parametrized direct query. drop procedure if exists ps_get_all_info_for_files_in_dir; DELIMITER // @@ -745,6 +750,7 @@ DELIMITER ; -- ps_get_all_info_for_file_ids : get all the info for given file ids -- file_ids : list of file ids -- output : FileID, Size, UID, GID, s.Status, GUID, CreationDate +-- DEPRECATED: FileManagerPs._getFileMetadataByID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_all_info_for_file_ids; DELIMITER // @@ -856,6 +862,7 @@ DELIMITER ; -- ps_get_file_ids_from_guids : return list of file ids for given guids -- guids : list of guids -- output : GUID, FileID +-- DEPRECATED: FileManagerPs._getFileIDFromGUID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_ids_from_guids; DELIMITER // @@ -875,6 +882,7 @@ DELIMITER ; -- ps_get_lfns_from_guids : return list of file lfns for given guids -- guids : list of guids -- output : GUID, LFN +-- DEPRECATED: FileManagerPs.getLFNForGUID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_lfns_from_guids; DELIMITER // @@ -1101,6 +1109,7 @@ DELIMITER ; -- file_id : file id -- se id : storage element id -- rep_id (out) : replica id +-- DEPRECATED: FileManagerPs._getRepIDsForReplica now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_replica_id; DELIMITER // @@ -1161,6 +1170,7 @@ DELIMITER ; -- status_id : new status id -- -- output : 0, number of column affected (should be 1 or 0), 'OK' +-- DEPRECATED: FileManagerPs._setReplicaStatus now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_replica_status; DELIMITER // @@ -1219,6 +1229,7 @@ DELIMITER ; -- in_uid : new uid -- -- output : errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_uid; DELIMITER // @@ -1240,6 +1251,7 @@ DELIMITER ; -- in_gid : new gid -- -- output : errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_gid; DELIMITER // @@ -1262,6 +1274,7 @@ DELIMITER ; -- status_id : id of the new status -- -- output errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_status; DELIMITER // @@ -1284,6 +1297,7 @@ DELIMITER ; -- in_mode : new mode -- -- output errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_mode; DELIMITER // @@ -1350,6 +1364,7 @@ DELIMITER ; -- visibleReplicaStatus : list of status we are interested in -- -- output : FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN +-- DEPRECATED: FileManagerPs._getFileReplicas now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_all_info_of_replicas_bulk; DELIMITER // @@ -1903,6 +1918,7 @@ DELIMITER ; -- ps_get_full_lfn_for_file_ids : get the full lfn for given file ids -- file_ids : list of file ids -- output : FileID, lfn +-- DEPRECATED: FileManagerPs._getFileLFNs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_full_lfn_for_file_ids; DELIMITER // @@ -1927,6 +1943,7 @@ DELIMITER ; -- ps_get_se_dump : dump all the lfns in list of SEs, with checksum and size -- se_id : storageElement IDs -- output : SEName, LFN, Checksum, Size +-- DEPRECATED: FileManagerPs.getSEDump now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_se_dump; DELIMITER // From 6ca66e6906605f2fb1af5dbdb63a2ef4a16d5333 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:00:29 +0200 Subject: [PATCH 02/17] refactor: Replace ps_find_dir with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 11c91afa598..67f50b04dfe 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -37,15 +37,24 @@ def findDir(self, path, connection=False): """ dpath = os.path.normpath(path) - result = self.db.executeStoredProcedure("ps_find_dir", (dpath, "ret1", "ret2"), outputIds=[1, 2]) + # TODO: Deprecated stored procedure ps_find_dir, replace with direct query + req = "SELECT DirID FROM FC_DirectoryList WHERE Name = %s" + result = self.db._query(req, args=(dpath,), conn=connection) if not result["OK"]: return result if not result["Value"]: return S_OK(0) - res = S_OK(result["Value"][0]) - res["Level"] = result["Value"][1] + dir_id = result["Value"][0][0] + req = "SELECT max(Depth) FROM FC_DirectoryClosure WHERE ChildID = %s" + result = self.db._query(req, args=(dir_id,), conn=connection) + if not result["OK"]: + return result + + depth = result["Value"][0][0] if result["Value"] else 0 + res = S_OK(dir_id) + res["Level"] = depth return res def findDirs(self, paths, connection=False): From 6caeaf7f9f2f7f620b9fdc2b3cd4e6fda734b255 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:01:10 +0200 Subject: [PATCH 03/17] refactor: Replace ps_find_dirs with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 67f50b04dfe..c633590e370 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -68,8 +68,12 @@ def findDirs(self, paths, connection=False): dirDict = {} if not paths: return S_OK(dirDict) - dpaths = stringListToString([os.path.normpath(path) for path in paths]) - result = self.db.executeStoredProcedureWithCursor("ps_find_dirs", (dpaths,)) + # TODO: Deprecated stored procedure ps_find_dirs, replace with direct query + normPaths = [os.path.normpath(path) for path in paths] + req = "SELECT Name, DirID FROM FC_DirectoryList WHERE Name IN (" + req += ",".join(["%s"] * len(normPaths)) + req += ")" + result = self.db._query(req, args=normPaths, conn=connection) if not result["OK"]: return result for dirName, dirID in result["Value"]: From 053f09340a6bc9d4297b7608c3a86cc2a38d7de8 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:01:51 +0200 Subject: [PATCH 04/17] refactor: Replace ps_remove_dir with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index c633590e370..4f8fe698c95 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -103,7 +103,9 @@ def removeDir(self, path): return res dirId = result["Value"] - result = self.db.executeStoredProcedure("ps_remove_dir", (dirId,), outputIds=[]) + # TODO: Deprecated stored procedure ps_remove_dir, replace with direct query + req = "DELETE FROM FC_DirectoryList WHERE DirID = %s" + result = self.db._update(req, args=(dirId,)) if not result["OK"]: return result From a039fa1a4c44f4ce7a7ffc03619d466ecc8e82bd Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:02:52 +0200 Subject: [PATCH 05/17] refactor: Replace ps_get_dirName_from_id with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 4f8fe698c95..34a4015654c 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -138,15 +138,16 @@ def getDirectoryPath(self, dirID): """ - result = self.db.executeStoredProcedure("ps_get_dirName_from_id", (dirID, "out"), outputIds=[1]) + # TODO: Deprecated stored procedure ps_get_dirName_from_id, replace with direct query + req = "SELECT Name FROM FC_DirectoryList WHERE DirID = %s" + result = self.db._query(req, args=(dirID,)) if not result["OK"]: return result - dirName = result["Value"][0] - - if not dirName: + if not result["Value"]: return S_ERROR("Directory with id %d not found" % int(dirID)) + dirName = result["Value"][0][0] return S_OK(dirName) def getDirectoryPaths(self, dirIDList): From 49f4b7ca94046f31d79e4b28c6927ee17835b1b8 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:04:08 +0200 Subject: [PATCH 06/17] refactor: Replace ps_get_dirNames_from_ids with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 34a4015654c..30753587bd2 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -163,9 +163,11 @@ def getDirectoryPaths(self, dirIDList): dirDict = {} - # Format the list - dIds = intListToString(dirs) - result = self.db.executeStoredProcedureWithCursor("ps_get_dirNames_from_ids", (dIds,)) + # TODO: Deprecated stored procedure ps_get_dirNames_from_ids, replace with direct query + req = "SELECT DirID, Name FROM FC_DirectoryList WHERE DirID IN (" + req += ",".join(["%s"] * len(dirs)) + req += ")" + result = self.db._query(req, args=dirs) if not result["OK"]: return result From a425031a090f3b2386add7a6bc719eb84ea057ad Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:05:03 +0200 Subject: [PATCH 07/17] refactor: Replace ps_get_parentIds_from_id with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 30753587bd2..97f4d606cb7 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -205,7 +205,9 @@ def getPathIDsByID(self, dirID): """ - result = self.db.executeStoredProcedureWithCursor("ps_get_parentIds_from_id", (dirID,)) + # TODO: Deprecated stored procedure ps_get_parentIds_from_id, replace with direct query + req = "SELECT ParentID FROM FC_DirectoryClosure WHERE ChildID = %s ORDER BY Depth DESC" + result = self.db._query(req, args=(dirID,)) if not result["OK"]: return result From 774010c7d552962b39207304c628b41de8f29009 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:05:40 +0200 Subject: [PATCH 08/17] refactor: Replace ps_get_direct_children with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 97f4d606cb7..d19548061e5 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -226,7 +226,9 @@ def getChildren(self, path, connection=False): else: dirID = path - result = self.db.executeStoredProcedureWithCursor("ps_get_direct_children", (dirID,)) + # TODO: Deprecated stored procedure ps_get_direct_children, replace with direct query + req = "SELECT ChildID FROM FC_DirectoryClosure WHERE ParentID = %s AND Depth = 1" + result = self.db._query(req, args=(dirID,), conn=connection) if not result["OK"]: return result if not result["Value"]: From 1a4c9b162910872e9bbfe821bd406435123965c3 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:06:23 +0200 Subject: [PATCH 09/17] refactor: Replace ps_get_sub_directories with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index d19548061e5..678012ed3fc 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -255,7 +255,16 @@ def getSubdirectoriesByID(self, dirID, requestString=False, includeParent=False) reqStr += " AND Depth != 0" return S_OK(reqStr) - result = self.db.executeStoredProcedureWithCursor("ps_get_sub_directories", (dirID, includeParent)) + # TODO: Deprecated stored procedure ps_get_sub_directories, replace with direct query + req = """SELECT c1.ChildID, max(c1.Depth) AS lvl + FROM FC_DirectoryClosure c1 + JOIN FC_DirectoryClosure c2 ON c1.ChildID = c2.ChildID + WHERE c2.ParentID = %s""" + args = [dirID] + if not includeParent: + req += " AND c2.Depth != 0" + req += " GROUP BY c1.ChildID" + result = self.db._query(req, args=args) if not result["OK"]: return result if not result["Value"]: From c15b0ae6aebdefe8d0d0de6644f1dbcd0db70deb Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:07:30 +0200 Subject: [PATCH 10/17] refactor: Replace ps_get_multiple_sub_directories with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 678012ed3fc..ec3cb53dc33 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -283,8 +283,11 @@ def getAllSubdirectoriesByID(self, dirIdList): if not isinstance(dirIdList, list): dirs = [dirIdList] - dIds = intListToString(dirs) - result = self.db.executeStoredProcedureWithCursor("ps_get_multiple_sub_directories", (dIds,)) + # TODO: Deprecated stored procedure ps_get_multiple_sub_directories, replace with direct query + req = "SELECT DISTINCT ChildID FROM FC_DirectoryClosure WHERE ParentID IN (" + req += ",".join(["%s"] * len(dirs)) + req += ")" + result = self.db._query(req, args=dirs) if not result["OK"]: return result From 756b26e51fe7c73642f32f07d07867e1614198c0 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:08:45 +0200 Subject: [PATCH 11/17] refactor: Replace ps_count_sub_directories with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index ec3cb53dc33..1073b2072b7 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -322,13 +322,17 @@ def countSubdirectories(self, dirId, includeParent=True): :returns: S_OK(value) """ - result = self.db.executeStoredProcedure( - "ps_count_sub_directories", (dirId, includeParent, "ret1"), outputIds=[2] - ) + # TODO: Deprecated stored procedure ps_count_sub_directories, replace with direct query + req = "SELECT count(ChildID) FROM FC_DirectoryClosure WHERE ParentID = %s" + result = self.db._query(req, args=(dirId,)) if not result["OK"]: return result - res = S_OK(result["Value"][0]) + count = result["Value"][0][0] if result["Value"] else 0 + # The stored procedure subtracts 1 if not includeParent + if not includeParent: + count = max(0, count - 1) + res = S_OK(count) return res ######################################################################################################## From ee8e96cde244ba5453a3aef2f811832918f5e2e3 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:09:52 +0200 Subject: [PATCH 12/17] refactor: Replace ps_get_all_directory_info and ps_get_all_directory_info_from_id with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 1073b2072b7..0086d4915ce 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -465,21 +465,7 @@ def getDirectoryParameters(self, pathOrDirId): :returns: S_OK(dict), where dict has the following keys: "DirID", "UID", "Owner", "GID", "OwnerGroup", "Status", "Mode", "CreationDate", "ModificationDate" """ - # Which procedure to use - psName = None - # it is a path ... - if isinstance(pathOrDirId, str): - psName = "ps_get_all_directory_info" - # it is the dirId - elif isinstance(pathOrDirId, ((list,) + (int,))): - psName = "ps_get_all_directory_info_from_id" - else: - return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") - - result = self.db.executeStoredProcedureWithCursor(psName, (pathOrDirId,)) - if not result["OK"]: - return result - + # TODO: Deprecated stored procedures ps_get_all_directory_info and ps_get_all_directory_info_from_id, replace with direct query # All the fields returned fieldNames = [ "DirID", @@ -493,6 +479,27 @@ def getDirectoryParameters(self, pathOrDirId): "ModificationDate", ] + # Build the query based on input type + req = """SELECT d.DirID, d.UID, u.UserName, d.GID, g.GroupName, d.Status, d.Mode, d.CreationDate, d.ModificationDate + FROM FC_DirectoryList d + JOIN FC_Users u ON d.UID = u.UID + JOIN FC_Groups g ON d.GID = g.GID + WHERE """ + if isinstance(pathOrDirId, str): + # TODO: Deprecated stored procedure ps_get_all_directory_info, replace with direct query + req += "d.Name = %s" + args = (pathOrDirId,) + elif isinstance(pathOrDirId, ((list,) + (int,))): + # TODO: Deprecated stored procedure ps_get_all_directory_info_from_id, replace with direct query + req += "d.DirID = %s" + args = (pathOrDirId,) + else: + return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") + + result = self.db._query(req, args=args) + if not result["OK"]: + return result + if not result["Value"]: return S_ERROR(f"Directory does not exist {pathOrDirId}") From c87157a7a603da1c7d22df70d950f8565867319b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:11:12 +0200 Subject: [PATCH 13/17] refactor: Replace ps_set_dir_* with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 71 ++++++++++++++++--- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 0086d4915ce..c8ad1f5c2d1 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -525,6 +525,9 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): S_ERROR if the directory does not exist """ + # TODO: Deprecated stored procedures ps_set_dir_uid, ps_set_dir_gid, ps_set_dir_status, ps_set_dir_mode + # and their recursive versions, replace with direct queries + # The PS associated with a given parameter psNames = { "UID": "ps_set_dir_uid", @@ -535,21 +538,71 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): psName = psNames.get(pname, None) - # If we have a recursive procedure and it is wanted, call it - if recursive and pname in ["UID", "GID", "Mode"] and psName: - psName += "_recursive" - # If there is an associated procedure, we go for it if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (path, pvalue)) + # Build the query based on whether it's recursive and which parameter + if recursive and pname in ["UID", "GID", "Mode"]: + # Recursive update: update directory and all its children + # First, get the directory ID + dirResult = self.findDir(path) + if not dirResult["OK"]: + return dirResult + dirId = dirResult["Value"] + if not dirId: + return S_ERROR(errno.ENOENT, f"Directory does not exist: {path}") + + # Update directories recursively + if pname == "UID": + req = """UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.UID = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + result = self.db._update(req, args=(pvalue, dirId)) + # Also update files + req_files = """UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.UID = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + resultFiles = self.db._update(req_files, args=(pvalue, dirId)) + elif pname == "GID": + req = """UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.GID = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + result = self.db._update(req, args=(pvalue, dirId)) + # Also update files + req_files = """UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.GID = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + resultFiles = self.db._update(req_files, args=(pvalue, dirId)) + else: # Mode + req = """UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.Mode = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + result = self.db._update(req, args=(pvalue, dirId)) + resultFiles = S_OK(0) # Mode doesn't apply to files + else: + # Non-recursive update + if pname == "UID": + req = "UPDATE FC_DirectoryList SET UID = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + elif pname == "GID": + req = "UPDATE FC_DirectoryList SET GID = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + elif pname == "Status": + req = "UPDATE FC_DirectoryList SET Status = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + elif pname == "Mode": + req = "UPDATE FC_DirectoryList SET Mode = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + result = self.db._update(req, args=(pvalue, path)) + resultFiles = S_OK(0) # Non-recursive doesn't update files if not result["OK"]: return result + if not resultFiles["OK"]: + return resultFiles - errno, affected, errMsg = result["Value"][0] - if errno: - return S_ERROR(errMsg) - + affected = result["Value"] + if not affected: # Either there were no changes, or the directory does not exist exists = self.existsDir(path).get("Value", {}).get("Exists") From 0d594c9e3ab28d334f6d40d10404e5f2d09874ef Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:12:21 +0200 Subject: [PATCH 14/17] refactor: Replace ps_get_directory_dump with direct sql queries in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index c8ad1f5c2d1..dcd051845fb 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -775,7 +775,18 @@ def _getDirectoryDump(self, path): if not dirID: return S_ERROR(errno.ENOENT, f"{path} does not exist") - result = self.db.executeStoredProcedureWithCursor("ps_get_directory_dump", (dirID,)) + # TODO: Deprecated stored procedure ps_get_directory_dump, replace with direct query + req = """(SELECT d.Name, NULL, d.CreationDate + FROM FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + WHERE c.ParentID = %s AND Depth != 0) + UNION ALL + (SELECT CONCAT(d.Name, '/', f.FileName), Size, f.CreationDate + FROM FC_Files f + JOIN FC_DirectoryList d ON f.DirID = d.DirID + JOIN FC_DirectoryClosure c ON c.ChildID = f.DirID + WHERE ParentID = %s)""" + result = self.db._query(req, args=(dirID, dirID)) if not result["OK"]: return result From 398c6d43fc834d98036e89f78830bedf94d19791 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 13:13:01 +0200 Subject: [PATCH 15/17] chore: Remove unused imports in DirectoryClosure Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../FileCatalogComponents/DirectoryManager/DirectoryClosure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index dcd051845fb..f6d037cd5d2 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -11,7 +11,6 @@ import os from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.List import intListToString, stringListToString from DIRAC.DataManagementSystem.DB.FileCatalogComponents.DirectoryManager.DirectoryTreeBase import DirectoryTreeBase From 28ffc0a5a9e0bf3d38f80e9fbb685b14d3089b2b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 24 Jul 2026 16:32:17 +0200 Subject: [PATCH 16/17] fix: Update files mode in recursive mode change Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../DirectoryManager/DirectoryClosure.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index f6d037cd5d2..296b89a2447 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -581,7 +581,12 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): SET d.Mode = %s, d.ModificationDate = UTC_TIMESTAMP() WHERE c.ParentID = %s""" result = self.db._update(req, args=(pvalue, dirId)) - resultFiles = S_OK(0) # Mode doesn't apply to files + # Also update files + req_files = """UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.Mode = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + resultFiles = self.db._update(req_files, args=(pvalue, dirId)) else: # Non-recursive update if pname == "UID": From bcf652d28a8ebf180a5246dd6c84f7458c05da59 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 3 Aug 2026 14:07:43 +0200 Subject: [PATCH 17/17] refactor chgrp and all --- .../DirectoryManager/DirectoryClosure.py | 125 ++++++++---------- 1 file changed, 57 insertions(+), 68 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 296b89a2447..47877b67a49 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -1,15 +1,18 @@ -""" DIRAC FileCatalog component representing a directory tree with - a closure table +"""DIRAC FileCatalog component representing a directory tree with +a closure table - General warning: when we return the number of affected row, if the values did not change - then they are not taken into account, so we might return "Dir does not exist" - while it does.... the timestamp update should prevent this to happen, however if - you do it several times within 1 second, then there will be no changed, and affected = 0 +General warning: when we return the number of affected row, if the values did not change + then they are not taken into account, so we might return "Dir does not exist" + while it does.... the timestamp update should prevent this to happen, however if + you do it several times within 1 second, then there will be no changed, and affected = 0 """ + import errno import os +import MySQLdb + from DIRAC import S_OK, S_ERROR from DIRAC.DataManagementSystem.DB.FileCatalogComponents.DirectoryManager.DirectoryTreeBase import DirectoryTreeBase @@ -389,7 +392,7 @@ def makeDirectory(self, path, credDict, status=1): result = self.db.ugManager.getUserAndGroupID(credDict) if not result["OK"]: return result - (l_uid, l_gid) = result["Value"] + l_uid, l_gid = result["Value"] # Find the ID of the parent res = self.findDir(parentDir) @@ -512,7 +515,6 @@ def getDirectoryParameters(self, pathOrDirId): def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): """Set a numerical directory parameter - Rem: the parent class has a more generic method, which is called in case we are given an unknown parameter @@ -539,74 +541,17 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): # If there is an associated procedure, we go for it if psName: - # Build the query based on whether it's recursive and which parameter if recursive and pname in ["UID", "GID", "Mode"]: - # Recursive update: update directory and all its children - # First, get the directory ID - dirResult = self.findDir(path) - if not dirResult["OK"]: - return dirResult - dirId = dirResult["Value"] - if not dirId: - return S_ERROR(errno.ENOENT, f"Directory does not exist: {path}") - - # Update directories recursively - if pname == "UID": - req = """UPDATE FC_DirectoryList d - JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID - SET d.UID = %s, d.ModificationDate = UTC_TIMESTAMP() - WHERE c.ParentID = %s""" - result = self.db._update(req, args=(pvalue, dirId)) - # Also update files - req_files = """UPDATE FC_Files f - JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID - SET f.UID = %s, f.ModificationDate = UTC_TIMESTAMP() - WHERE c.ParentID = %s""" - resultFiles = self.db._update(req_files, args=(pvalue, dirId)) - elif pname == "GID": - req = """UPDATE FC_DirectoryList d - JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID - SET d.GID = %s, d.ModificationDate = UTC_TIMESTAMP() - WHERE c.ParentID = %s""" - result = self.db._update(req, args=(pvalue, dirId)) - # Also update files - req_files = """UPDATE FC_Files f - JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID - SET f.GID = %s, f.ModificationDate = UTC_TIMESTAMP() - WHERE c.ParentID = %s""" - resultFiles = self.db._update(req_files, args=(pvalue, dirId)) - else: # Mode - req = """UPDATE FC_DirectoryList d - JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID - SET d.Mode = %s, d.ModificationDate = UTC_TIMESTAMP() - WHERE c.ParentID = %s""" - result = self.db._update(req, args=(pvalue, dirId)) - # Also update files - req_files = """UPDATE FC_Files f - JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID - SET f.Mode = %s, f.ModificationDate = UTC_TIMESTAMP() - WHERE c.ParentID = %s""" - resultFiles = self.db._update(req_files, args=(pvalue, dirId)) + result = self._setDirectoryParameterRecursively(path, pname, pvalue) else: - # Non-recursive update - if pname == "UID": - req = "UPDATE FC_DirectoryList SET UID = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" - elif pname == "GID": - req = "UPDATE FC_DirectoryList SET GID = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" - elif pname == "Status": - req = "UPDATE FC_DirectoryList SET Status = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" - elif pname == "Mode": - req = "UPDATE FC_DirectoryList SET Mode = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + req = f"UPDATE FC_DirectoryList SET {pname} = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" # nosec B608 result = self.db._update(req, args=(pvalue, path)) - resultFiles = S_OK(0) # Non-recursive doesn't update files if not result["OK"]: return result - if not resultFiles["OK"]: - return resultFiles affected = result["Value"] - + if not affected: # Either there were no changes, or the directory does not exist exists = self.existsDir(path).get("Value", {}).get("Exists") @@ -620,6 +565,50 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): else: return DirectoryTreeBase._setDirectoryParameter(self, path, pname, pvalue) + def _setDirectoryParameterRecursively(self, path, pname, pvalue): + """Set a directory parameter on a directory tree and its files atomically.""" + + result = self.db._getConnection() + if not result["OK"]: + return result + connection = result["Value"] + + directory_query = f"""UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.{pname} = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" # nosec B608 + file_query = f"""UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.{pname} = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" # nosec B608 + + with connection.cursor() as cursor: + try: + cursor.execute("START TRANSACTION") + cursor.execute( + "SELECT DirID FROM FC_DirectoryList WHERE Name = %s", + (os.path.normpath(path),), + ) + row = cursor.fetchone() + if not row: + cursor.execute("ROLLBACK") + return S_ERROR(errno.ENOENT, f"Directory does not exist: {path}") + + dir_id = row[0] + cursor.execute(directory_query, (pvalue, dir_id)) + directory_updates = cursor.rowcount + cursor.execute(file_query, (pvalue, dir_id)) + file_updates = cursor.rowcount + cursor.execute("COMMIT") + except MySQLdb.Error as error: + try: + cursor.execute("ROLLBACK") + except MySQLdb.Error as rollback_error: + return S_ERROR(f"Recursive directory update failed: {error}; rollback failed: {rollback_error}") + return S_ERROR(f"Recursive directory update failed: {error}") + + return S_OK(directory_updates + file_updates) + def _setDirectoryGroup(self, path, gname, recursive=False): """Set the directory owner"""