diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 11c91afa598..47877b67a49 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -1,17 +1,19 @@ -""" DIRAC FileCatalog component representing a directory tree with - a closure table +"""DIRAC FileCatalog component representing a directory tree with +a closure table - General warning: when we return the number of affected row, if the values did not change - then they are not taken into account, so we might return "Dir does not exist" - while it does.... the timestamp update should prevent this to happen, however if - you do it several times within 1 second, then there will be no changed, and affected = 0 +General warning: when we return the number of affected row, if the values did not change + then they are not taken into account, so we might return "Dir does not exist" + while it does.... the timestamp update should prevent this to happen, however if + you do it several times within 1 second, then there will be no changed, and affected = 0 """ + import errno import os +import MySQLdb + from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.List import intListToString, stringListToString from DIRAC.DataManagementSystem.DB.FileCatalogComponents.DirectoryManager.DirectoryTreeBase import DirectoryTreeBase @@ -37,15 +39,24 @@ def findDir(self, path, connection=False): """ dpath = os.path.normpath(path) - result = self.db.executeStoredProcedure("ps_find_dir", (dpath, "ret1", "ret2"), outputIds=[1, 2]) + # TODO: Deprecated stored procedure ps_find_dir, replace with direct query + req = "SELECT DirID FROM FC_DirectoryList WHERE Name = %s" + result = self.db._query(req, args=(dpath,), conn=connection) if not result["OK"]: return result if not result["Value"]: return S_OK(0) - res = S_OK(result["Value"][0]) - res["Level"] = result["Value"][1] + dir_id = result["Value"][0][0] + req = "SELECT max(Depth) FROM FC_DirectoryClosure WHERE ChildID = %s" + result = self.db._query(req, args=(dir_id,), conn=connection) + if not result["OK"]: + return result + + depth = result["Value"][0][0] if result["Value"] else 0 + res = S_OK(dir_id) + res["Level"] = depth return res def findDirs(self, paths, connection=False): @@ -59,8 +70,12 @@ def findDirs(self, paths, connection=False): dirDict = {} if not paths: return S_OK(dirDict) - dpaths = stringListToString([os.path.normpath(path) for path in paths]) - result = self.db.executeStoredProcedureWithCursor("ps_find_dirs", (dpaths,)) + # TODO: Deprecated stored procedure ps_find_dirs, replace with direct query + normPaths = [os.path.normpath(path) for path in paths] + req = "SELECT Name, DirID FROM FC_DirectoryList WHERE Name IN (" + req += ",".join(["%s"] * len(normPaths)) + req += ")" + result = self.db._query(req, args=normPaths, conn=connection) if not result["OK"]: return result for dirName, dirID in result["Value"]: @@ -90,7 +105,9 @@ def removeDir(self, path): return res dirId = result["Value"] - result = self.db.executeStoredProcedure("ps_remove_dir", (dirId,), outputIds=[]) + # TODO: Deprecated stored procedure ps_remove_dir, replace with direct query + req = "DELETE FROM FC_DirectoryList WHERE DirID = %s" + result = self.db._update(req, args=(dirId,)) if not result["OK"]: return result @@ -123,15 +140,16 @@ def getDirectoryPath(self, dirID): """ - result = self.db.executeStoredProcedure("ps_get_dirName_from_id", (dirID, "out"), outputIds=[1]) + # TODO: Deprecated stored procedure ps_get_dirName_from_id, replace with direct query + req = "SELECT Name FROM FC_DirectoryList WHERE DirID = %s" + result = self.db._query(req, args=(dirID,)) if not result["OK"]: return result - dirName = result["Value"][0] - - if not dirName: + if not result["Value"]: return S_ERROR("Directory with id %d not found" % int(dirID)) + dirName = result["Value"][0][0] return S_OK(dirName) def getDirectoryPaths(self, dirIDList): @@ -147,9 +165,11 @@ def getDirectoryPaths(self, dirIDList): dirDict = {} - # Format the list - dIds = intListToString(dirs) - result = self.db.executeStoredProcedureWithCursor("ps_get_dirNames_from_ids", (dIds,)) + # TODO: Deprecated stored procedure ps_get_dirNames_from_ids, replace with direct query + req = "SELECT DirID, Name FROM FC_DirectoryList WHERE DirID IN (" + req += ",".join(["%s"] * len(dirs)) + req += ")" + result = self.db._query(req, args=dirs) if not result["OK"]: return result @@ -187,7 +207,9 @@ def getPathIDsByID(self, dirID): """ - result = self.db.executeStoredProcedureWithCursor("ps_get_parentIds_from_id", (dirID,)) + # TODO: Deprecated stored procedure ps_get_parentIds_from_id, replace with direct query + req = "SELECT ParentID FROM FC_DirectoryClosure WHERE ChildID = %s ORDER BY Depth DESC" + result = self.db._query(req, args=(dirID,)) if not result["OK"]: return result @@ -206,7 +228,9 @@ def getChildren(self, path, connection=False): else: dirID = path - result = self.db.executeStoredProcedureWithCursor("ps_get_direct_children", (dirID,)) + # TODO: Deprecated stored procedure ps_get_direct_children, replace with direct query + req = "SELECT ChildID FROM FC_DirectoryClosure WHERE ParentID = %s AND Depth = 1" + result = self.db._query(req, args=(dirID,), conn=connection) if not result["OK"]: return result if not result["Value"]: @@ -233,7 +257,16 @@ def getSubdirectoriesByID(self, dirID, requestString=False, includeParent=False) reqStr += " AND Depth != 0" return S_OK(reqStr) - result = self.db.executeStoredProcedureWithCursor("ps_get_sub_directories", (dirID, includeParent)) + # TODO: Deprecated stored procedure ps_get_sub_directories, replace with direct query + req = """SELECT c1.ChildID, max(c1.Depth) AS lvl + FROM FC_DirectoryClosure c1 + JOIN FC_DirectoryClosure c2 ON c1.ChildID = c2.ChildID + WHERE c2.ParentID = %s""" + args = [dirID] + if not includeParent: + req += " AND c2.Depth != 0" + req += " GROUP BY c1.ChildID" + result = self.db._query(req, args=args) if not result["OK"]: return result if not result["Value"]: @@ -252,8 +285,11 @@ def getAllSubdirectoriesByID(self, dirIdList): if not isinstance(dirIdList, list): dirs = [dirIdList] - dIds = intListToString(dirs) - result = self.db.executeStoredProcedureWithCursor("ps_get_multiple_sub_directories", (dIds,)) + # TODO: Deprecated stored procedure ps_get_multiple_sub_directories, replace with direct query + req = "SELECT DISTINCT ChildID FROM FC_DirectoryClosure WHERE ParentID IN (" + req += ",".join(["%s"] * len(dirs)) + req += ")" + result = self.db._query(req, args=dirs) if not result["OK"]: return result @@ -288,13 +324,17 @@ def countSubdirectories(self, dirId, includeParent=True): :returns: S_OK(value) """ - result = self.db.executeStoredProcedure( - "ps_count_sub_directories", (dirId, includeParent, "ret1"), outputIds=[2] - ) + # TODO: Deprecated stored procedure ps_count_sub_directories, replace with direct query + req = "SELECT count(ChildID) FROM FC_DirectoryClosure WHERE ParentID = %s" + result = self.db._query(req, args=(dirId,)) if not result["OK"]: return result - res = S_OK(result["Value"][0]) + count = result["Value"][0][0] if result["Value"] else 0 + # The stored procedure subtracts 1 if not includeParent + if not includeParent: + count = max(0, count - 1) + res = S_OK(count) return res ######################################################################################################## @@ -352,7 +392,7 @@ def makeDirectory(self, path, credDict, status=1): result = self.db.ugManager.getUserAndGroupID(credDict) if not result["OK"]: return result - (l_uid, l_gid) = result["Value"] + l_uid, l_gid = result["Value"] # Find the ID of the parent res = self.findDir(parentDir) @@ -427,21 +467,7 @@ def getDirectoryParameters(self, pathOrDirId): :returns: S_OK(dict), where dict has the following keys: "DirID", "UID", "Owner", "GID", "OwnerGroup", "Status", "Mode", "CreationDate", "ModificationDate" """ - # Which procedure to use - psName = None - # it is a path ... - if isinstance(pathOrDirId, str): - psName = "ps_get_all_directory_info" - # it is the dirId - elif isinstance(pathOrDirId, ((list,) + (int,))): - psName = "ps_get_all_directory_info_from_id" - else: - return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") - - result = self.db.executeStoredProcedureWithCursor(psName, (pathOrDirId,)) - if not result["OK"]: - return result - + # TODO: Deprecated stored procedures ps_get_all_directory_info and ps_get_all_directory_info_from_id, replace with direct query # All the fields returned fieldNames = [ "DirID", @@ -455,6 +481,27 @@ def getDirectoryParameters(self, pathOrDirId): "ModificationDate", ] + # Build the query based on input type + req = """SELECT d.DirID, d.UID, u.UserName, d.GID, g.GroupName, d.Status, d.Mode, d.CreationDate, d.ModificationDate + FROM FC_DirectoryList d + JOIN FC_Users u ON d.UID = u.UID + JOIN FC_Groups g ON d.GID = g.GID + WHERE """ + if isinstance(pathOrDirId, str): + # TODO: Deprecated stored procedure ps_get_all_directory_info, replace with direct query + req += "d.Name = %s" + args = (pathOrDirId,) + elif isinstance(pathOrDirId, ((list,) + (int,))): + # TODO: Deprecated stored procedure ps_get_all_directory_info_from_id, replace with direct query + req += "d.DirID = %s" + args = (pathOrDirId,) + else: + return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") + + result = self.db._query(req, args=args) + if not result["OK"]: + return result + if not result["Value"]: return S_ERROR(f"Directory does not exist {pathOrDirId}") @@ -468,7 +515,6 @@ def getDirectoryParameters(self, pathOrDirId): def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): """Set a numerical directory parameter - Rem: the parent class has a more generic method, which is called in case we are given an unknown parameter @@ -480,6 +526,9 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): S_ERROR if the directory does not exist """ + # TODO: Deprecated stored procedures ps_set_dir_uid, ps_set_dir_gid, ps_set_dir_status, ps_set_dir_mode + # and their recursive versions, replace with direct queries + # The PS associated with a given parameter psNames = { "UID": "ps_set_dir_uid", @@ -490,20 +539,18 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): psName = psNames.get(pname, None) - # If we have a recursive procedure and it is wanted, call it - if recursive and pname in ["UID", "GID", "Mode"] and psName: - psName += "_recursive" - # If there is an associated procedure, we go for it if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (path, pvalue)) + if recursive and pname in ["UID", "GID", "Mode"]: + result = self._setDirectoryParameterRecursively(path, pname, pvalue) + else: + req = f"UPDATE FC_DirectoryList SET {pname} = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" # nosec B608 + result = self.db._update(req, args=(pvalue, path)) if not result["OK"]: return result - errno, affected, errMsg = result["Value"][0] - if errno: - return S_ERROR(errMsg) + affected = result["Value"] if not affected: # Either there were no changes, or the directory does not exist @@ -518,6 +565,50 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): else: return DirectoryTreeBase._setDirectoryParameter(self, path, pname, pvalue) + def _setDirectoryParameterRecursively(self, path, pname, pvalue): + """Set a directory parameter on a directory tree and its files atomically.""" + + result = self.db._getConnection() + if not result["OK"]: + return result + connection = result["Value"] + + directory_query = f"""UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.{pname} = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" # nosec B608 + file_query = f"""UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.{pname} = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" # nosec B608 + + with connection.cursor() as cursor: + try: + cursor.execute("START TRANSACTION") + cursor.execute( + "SELECT DirID FROM FC_DirectoryList WHERE Name = %s", + (os.path.normpath(path),), + ) + row = cursor.fetchone() + if not row: + cursor.execute("ROLLBACK") + return S_ERROR(errno.ENOENT, f"Directory does not exist: {path}") + + dir_id = row[0] + cursor.execute(directory_query, (pvalue, dir_id)) + directory_updates = cursor.rowcount + cursor.execute(file_query, (pvalue, dir_id)) + file_updates = cursor.rowcount + cursor.execute("COMMIT") + except MySQLdb.Error as error: + try: + cursor.execute("ROLLBACK") + except MySQLdb.Error as rollback_error: + return S_ERROR(f"Recursive directory update failed: {error}; rollback failed: {rollback_error}") + return S_ERROR(f"Recursive directory update failed: {error}") + + return S_OK(directory_updates + file_updates) + def _setDirectoryGroup(self, path, gname, recursive=False): """Set the directory owner""" @@ -677,7 +768,18 @@ def _getDirectoryDump(self, path): if not dirID: return S_ERROR(errno.ENOENT, f"{path} does not exist") - result = self.db.executeStoredProcedureWithCursor("ps_get_directory_dump", (dirID,)) + # TODO: Deprecated stored procedure ps_get_directory_dump, replace with direct query + req = """(SELECT d.Name, NULL, d.CreationDate + FROM FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + WHERE c.ParentID = %s AND Depth != 0) + UNION ALL + (SELECT CONCAT(d.Name, '/', f.FileName), Size, f.CreationDate + FROM FC_Files f + JOIN FC_DirectoryList d ON f.DirID = d.DirID + JOIN FC_DirectoryClosure c ON c.ChildID = f.DirID + WHERE ParentID = %s)""" + result = self.db._query(req, args=(dirID, dirID)) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index b9204bb6449..bc166fdf78f 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -4,7 +4,7 @@ from DIRAC import S_OK, S_ERROR from DIRAC.DataManagementSystem.DB.FileCatalogComponents.FileManager.FileManagerBase import FileManagerBase -from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks +from DIRAC.Core.Utilities.List import intListToString, breakListIntoChunks from DIRAC.Core.Utilities.TimeUtilities import DiracTime # The logic of some methods is basically a copy/paste from the FileManager class, @@ -67,13 +67,17 @@ def _findFileIDs(self, lfns, connection=False): if len(lfns) == 1: lfn = list(lfns)[0] # if lfns is a dict, list(lfns) returns lfns.keys() pathPart, filePart = os.path.split(lfn) - result = self.db.executeStoredProcedure( - "ps_get_file_id_from_lfn", (pathPart, filePart, "ret1"), outputIds=[2] + req = ( + "SELECT f.FileID " + "FROM FC_Files f " + "JOIN FC_DirectoryList d ON d.DirID = f.DirID " + "WHERE d.Name = %s AND f.FileName = %s" ) + result = self.db._query(req, args=(pathPart, filePart), conn=connection) if not result["OK"]: return result - fileId = result["Value"][0] + fileId = result["Value"][0][0] if result["Value"] else 0 if not fileId: failed[lfn] = "No such file or directory" @@ -95,11 +99,10 @@ def _findFileIDs(self, lfns, connection=False): fileNames = filesInDirDict[dirPath] dirID = directoryPathToIds[dirPath] - formatedFileNames = stringListToString(fileNames) - - result = self.db.executeStoredProcedureWithCursor( - "ps_get_file_ids_from_dir_id", (dirID, formatedFileNames) - ) + req = "SELECT FileID, FileName FROM FC_Files WHERE DirID = %s AND FileName IN (" + req += ",".join(["%s"] * len(fileNames)) + req += ")" + result = self.db._query(req, args=(dirID, *fileNames), conn=connection) if not result["OK"]: return result for fileID, fileName in result["Value"]: @@ -135,14 +138,26 @@ def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, if "FileID" not in metadata: metadata.append("FileID") - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileNames = stringListToString(fileNames) - fStatus = stringListToString(self.db.visibleFileStatus) + req = """SELECT FileName, DirID, f.FileID, Size, f.uid, UserName, f.gid, GroupName, s.Status, + GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode + FROM FC_Files f + JOIN FC_Users u ON f.UID = u.UID + JOIN FC_Groups g ON f.GID = g.GID + JOIN FC_Statuses s ON f.Status = s.StatusID + WHERE DirID = %s""" + args = [dirID] + if not allStatus: + req += " AND s.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleFileStatus)) + req += ")" + args.extend(self.db.visibleFileStatus) + if fileNames: + req += " AND f.FileName IN (" + req += ",".join(["%s"] * len(fileNames)) + req += ")" + args.extend(fileNames) - specificFiles = True if len(fileNames) else False - result = self.db.executeStoredProcedureWithCursor( - "ps_get_all_info_for_files_in_dir", (dirID, specificFiles, formatedFileNames, allStatus, fStatus) - ) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -187,9 +202,13 @@ def _getFileMetadataByID(self, fileIDs, connection=False): ["FileID", "Size", "UID", "GID", "s.Status", "GUID", "CreationDate"] """ - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileIds = intListToString(fileIDs) - result = self.db.executeStoredProcedureWithCursor("ps_get_all_info_for_file_ids", (formatedFileIds,)) + req = """SELECT f.FileID, Size, UID, GID, s.Status, GUID, CreationDate + FROM FC_Files f + JOIN FC_Statuses s ON f.Status = s.StatusID + WHERE f.FileID IN (""" + req += ",".join(["%s"] * len(fileIDs)) + req += ")" + result = self.db._query(req, args=fileIDs, conn=connection) if not result["OK"]: return result @@ -346,9 +365,10 @@ def _getFileIDFromGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] - # formatedGuids = ','.join( [ '"%s"' % guid for guid in guids ] ) - formatedGuids = stringListToString(guids) - result = self.db.executeStoredProcedureWithCursor("ps_get_file_ids_from_guids", (formatedGuids,)) + req = "SELECT GUID, FileID FROM FC_Files WHERE GUID IN (" + req += ",".join(["%s"] * len(guids)) + req += ")" + result = self.db._query(req, args=guids, conn=connection) if not result["OK"]: return result @@ -366,8 +386,15 @@ def getLFNForGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] - formatedGuids = stringListToString(guids) - result = self.db.executeStoredProcedureWithCursor("ps_get_lfns_from_guids", (formatedGuids,)) + req = ( + 'SELECT GUID, CONCAT(d.Name, "/", f.FileName) ' + "FROM FC_Files f " + "JOIN FC_DirectoryList d on f.DirID = d.DirID " + "WHERE GUID IN (" + ) + req += ",".join(["%s"] * len(guids)) + req += ")" + result = self.db._query(req, args=guids, conn=connection) if not result["OK"]: return result @@ -578,16 +605,19 @@ def _getRepIDsForReplica(self, replicaTuples, connection=False): replicaDict = {} - for fileID, seID in replicaTuples: - result = self.db.executeStoredProcedure("ps_get_replica_id", (fileID, seID, "repIdOut"), outputIds=[2]) - if not result["OK"]: - return result + if not replicaTuples: + return S_OK(replicaDict) - repID = result["Value"][0] + req = "SELECT RepID, FileID, SEID FROM FC_Replicas WHERE (FileID, SEID) IN (" + req += ",".join(["(%s,%s)"] * len(replicaTuples)) + req += ")" + args = tuple(value for replicaTuple in replicaTuples for value in replicaTuple) + result = self.db._query(req, args=args, conn=connection) + if not result["OK"]: + return result - # if the replica exists, we add it to the dict - if repID: - replicaDict.setdefault(fileID, {}).setdefault(seID, repID) + for repID, fileID, seID in result["Value"]: + replicaDict.setdefault(fileID, {}).setdefault(seID, repID) return S_OK(replicaDict) @@ -674,11 +704,12 @@ def _setReplicaStatus(self, fileID, se, status, connection=False): return res seID = res["Value"] - result = self.db.executeStoredProcedureWithCursor("ps_set_replica_status", (fileID, seID, statusID)) + req = "UPDATE FC_Replicas SET Status = %s WHERE FileID = %s AND SEID = %s" + result = self.db._update(req, args=(statusID, fileID, seID), conn=connection) if not result["OK"]: return result - affected = result["Value"][0][0] # Affected is the number of raws updated + affected = result["Value"] if not affected: return S_ERROR("Replica does not exist") @@ -734,36 +765,18 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): """ connection = self._getConnection(connection) - # The PS associated with a given parameter - psNames = { - "UID": "ps_set_file_uid", - "GID": "ps_set_file_gid", - "Status": "ps_set_file_status", - "Mode": "ps_set_file_mode", - } - - psName = psNames.get(paramName, None) - - # If there is an associated procedure, we go for it - if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (fileID, paramValue)) - if not result["OK"]: - return result - - _affected = result["Value"][0][0] - # If affected = 0, the file does not exist, but who cares... - - # In case this is a 'new' parameter, we have a failback solution, but we - # should add a specific ps for it - else: - if not self.db._checkIdentifier(paramName)["OK"]: - return S_ERROR(f"ParamName is invalid: {paramName}") - req = "UPDATE FC_Files SET " - req += f"{paramName}=%s, ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (" - req += ",".join(["%s"] * len(fileID)) - req += ")" - args = [paramValue] + fileID - return self.db._update(req, args=args, conn=connection) + if not isinstance(fileID, (list, tuple)): + fileID = [fileID] + if not self.db._checkIdentifier(paramName)["OK"]: + return S_ERROR(f"ParamName is invalid: {paramName}") + req = "UPDATE FC_Files SET " + req += f"{paramName}=%s, ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (" + req += ",".join(["%s"] * len(fileID)) + req += ")" + args = [paramValue] + fileID + result = self.db._update(req, args=args, conn=connection) + if not result["OK"]: + return result return S_OK() @@ -796,17 +809,23 @@ def _getFileReplicas(self, fileIDs, fields_input=None, allStatus=False, connecti # non existing replicas replicas = {fileID: {} for fileID in fileIDs} - # Format the status to be used in a IN clause in the stored procedure - fStatus = stringListToString(self.db.visibleReplicaStatus) - fieldNames = ["FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"] for chunks in breakListIntoChunks(fileIDs, 1000): - # Format the FileIDs to be used in a IN clause in the stored procedure - formatedFileIds = intListToString(chunks) - result = self.db.executeStoredProcedureWithCursor( - "ps_get_all_info_of_replicas_bulk", (formatedFileIds, allStatus, fStatus) - ) + req = """SELECT FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN + FROM FC_Replicas r + JOIN FC_StorageElements se on r.SEID = se.SEID + JOIN FC_Statuses st on r.Status = st.StatusID + WHERE FileID IN (""" + req += ",".join(["%s"] * len(chunks)) + req += ")" + args = list(chunks) + if not allStatus: + req += " AND st.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleReplicaStatus)) + req += ")" + args.extend(self.db.visibleReplicaStatus) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -829,11 +848,12 @@ def countFilesInDir(self, dirId): :returns: S_OK(value) or S_ERROR """ - result = self.db.executeStoredProcedure("ps_count_files_in_dir", (dirId, "ret1"), outputIds=[1]) + req = "SELECT count(FileID) FROM FC_Files WHERE DirID = %s" + result = self.db._query(req, args=(dirId,)) if not result["OK"]: return result - res = S_OK(result["Value"][0]) + res = S_OK(result["Value"][0][0] if result["Value"] else 0) return res ########################################################################## @@ -880,16 +900,24 @@ def getDirectoryReplicas(self, dirID, path, allStatus=False, connection=False): values from the configuration """ - # We format the visible file/replica satus so we can give it as argument to the ps - # It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'-- - # fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] ) - # rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] ) - fStatus = stringListToString(self.db.visibleFileStatus) - rStatus = stringListToString(self.db.visibleReplicaStatus) + req = """SELECT f.FileName, f.FileID, s.SEName, r.PFN + FROM FC_Replicas r + JOIN FC_Files f on f.FileID = r.FileID + JOIN FC_StorageElements s on s.SEID = r.SEID + JOIN FC_Statuses fst on f.Status = fst.StatusID + JOIN FC_Statuses rst on r.Status = rst.StatusID + WHERE DirID = %s""" + args = [dirID] + if not allStatus: + req += " AND fst.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleFileStatus)) + req += ") AND rst.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleReplicaStatus)) + req += ")" + args.extend(self.db.visibleFileStatus) + args.extend(self.db.visibleReplicaStatus) - result = self.db.executeStoredProcedureWithCursor( - "ps_get_replicas_for_files_in_dir", (dirID, allStatus, fStatus, rStatus) - ) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -906,9 +934,13 @@ def _getFileLFNs(self, fileIDs): successful = {} for chunks in breakListIntoChunks(fileIDs, 1000): - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileIds = intListToString(chunks) - result = self.db.executeStoredProcedureWithCursor("ps_get_full_lfn_for_file_ids", (formatedFileIds,)) + req = """SELECT f.FileID, CONCAT(d.Name, "/", f.FileName) + FROM FC_Files f + JOIN FC_DirectoryList d ON f.DirID = d.DirID + WHERE f.FileID IN (""" + req += ",".join(["%s"] * len(chunks)) + req += ")" + result = self.db._query(req, args=chunks) if not result["OK"]: return result @@ -938,6 +970,13 @@ def getSEDump(self, seNames): return res seIDs.append(res["Value"]) - formatedSEIds = intListToString(seIDs) + req = """SELECT SEName, CONCAT(d.Name, "/", f.FileName), f.Checksum, f.Size + FROM FC_Files f + JOIN FC_Replicas r on f.FileID = r.FileID + JOIN FC_DirectoryList d on d.DirID = f.DirID + JOIN FC_StorageElements s on r.SEID = s.SEID + WHERE s.SEID IN (""" + req += ",".join(["%s"] * len(seIDs)) + req += ")" - return self.db.executeStoredProcedureWithCursor("ps_get_se_dump", (formatedSEIds,)) + return self.db._query(req, args=seIDs) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql index 0c00503540d..e0524bab338 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql @@ -565,6 +565,7 @@ DELIMITER ; -- ps_count_files_in_dir : counts how many files are in a given directory -- dir_id : directory id -- countFile (out value): amount of files +-- DEPRECATED: FileManagerPs.countFilesInDir now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_count_files_in_dir; DELIMITER // @@ -634,6 +635,7 @@ DELIMITER ; -- visibleFileStatus : status of files to be considered -- visibleReplicaStatus : status of replicas to be considered -- outputs : FileName, FileID, SEName, PFN +-- DEPRECATED: FileManagerPs.getDirectoryReplicas now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_replicas_for_files_in_dir; DELIMITER // @@ -661,6 +663,7 @@ DELIMITER ; -- dirName : name of the directory -- fileName : name of the file -- file_id (out) : id of the file +-- DEPRECATED: FileManagerPs._findFileIDs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_id_from_lfn; DELIMITER // @@ -683,6 +686,7 @@ DELIMITER ; -- dir_id : directory id -- file_names : names of the files we are interested in -- output : FileID, FileName +-- DEPRECATED: FileManagerPs._findFileIDs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_ids_from_dir_id; DELIMITER // @@ -710,6 +714,7 @@ DELIMITER ; -- visibleFileStatus : list of status we are interested in -- output : FileName, DirID, f.FileID, Size, f.uid, UserName, f.gid, GroupName, s.Status, -- GUID, Checksum, ChecksumType, Type, CreationDate,ModificationDate, Mode +-- DEPRECATED: FileManagerPs._getDirectoryFiles now uses a parametrized direct query. drop procedure if exists ps_get_all_info_for_files_in_dir; DELIMITER // @@ -745,6 +750,7 @@ DELIMITER ; -- ps_get_all_info_for_file_ids : get all the info for given file ids -- file_ids : list of file ids -- output : FileID, Size, UID, GID, s.Status, GUID, CreationDate +-- DEPRECATED: FileManagerPs._getFileMetadataByID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_all_info_for_file_ids; DELIMITER // @@ -856,6 +862,7 @@ DELIMITER ; -- ps_get_file_ids_from_guids : return list of file ids for given guids -- guids : list of guids -- output : GUID, FileID +-- DEPRECATED: FileManagerPs._getFileIDFromGUID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_ids_from_guids; DELIMITER // @@ -875,6 +882,7 @@ DELIMITER ; -- ps_get_lfns_from_guids : return list of file lfns for given guids -- guids : list of guids -- output : GUID, LFN +-- DEPRECATED: FileManagerPs.getLFNForGUID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_lfns_from_guids; DELIMITER // @@ -1101,6 +1109,7 @@ DELIMITER ; -- file_id : file id -- se id : storage element id -- rep_id (out) : replica id +-- DEPRECATED: FileManagerPs._getRepIDsForReplica now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_replica_id; DELIMITER // @@ -1161,6 +1170,7 @@ DELIMITER ; -- status_id : new status id -- -- output : 0, number of column affected (should be 1 or 0), 'OK' +-- DEPRECATED: FileManagerPs._setReplicaStatus now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_replica_status; DELIMITER // @@ -1219,6 +1229,7 @@ DELIMITER ; -- in_uid : new uid -- -- output : errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_uid; DELIMITER // @@ -1240,6 +1251,7 @@ DELIMITER ; -- in_gid : new gid -- -- output : errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_gid; DELIMITER // @@ -1262,6 +1274,7 @@ DELIMITER ; -- status_id : id of the new status -- -- output errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_status; DELIMITER // @@ -1284,6 +1297,7 @@ DELIMITER ; -- in_mode : new mode -- -- output errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_mode; DELIMITER // @@ -1350,6 +1364,7 @@ DELIMITER ; -- visibleReplicaStatus : list of status we are interested in -- -- output : FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN +-- DEPRECATED: FileManagerPs._getFileReplicas now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_all_info_of_replicas_bulk; DELIMITER // @@ -1903,6 +1918,7 @@ DELIMITER ; -- ps_get_full_lfn_for_file_ids : get the full lfn for given file ids -- file_ids : list of file ids -- output : FileID, lfn +-- DEPRECATED: FileManagerPs._getFileLFNs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_full_lfn_for_file_ids; DELIMITER // @@ -1927,6 +1943,7 @@ DELIMITER ; -- ps_get_se_dump : dump all the lfns in list of SEs, with checksum and size -- se_id : storageElement IDs -- output : SEName, LFN, Checksum, Size +-- DEPRECATED: FileManagerPs.getSEDump now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_se_dump; DELIMITER //