Changeset 37:a9637d8e2ea0 for src/mlx

Timestamp:

03/04/12 08:58:27 (13 years ago)

Author:

István Váradi <ivaradi@…>

Branch:

default

Phase:

public

Message:

The MLXMANIFEST is generated when creating the distribution and files to be removed are really removed.

Location:

src/mlx

Files:

• gui/update.py (modified) (3 diffs)
• update.py (modified) (15 diffs)

src/mlx/gui/update.py

@@ -157 +157 @@
             self._sudoCondition.notify()
         
-    def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove):
+    def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
+                     numToRemoveLocal):
         """Called when the downloading of the files has started."""
         self._numToModifyAndNew = numToModifyAndNew
@@ -164 +165 @@
         self._downloaded = 0
         self._numToRemove = numToRemove
+        self._numToRemoveLocal = numToRemoveLocal
         self._numRemoved = 0
 
         self._totalProgress = self._totalSize + \
-                              (self._numToModifyAndNew + self._numToRemove) * \
+                              (self._numToModifyAndNew + \
+                               self._numToRemove + self._numToRemoveLocal) * \
                               Updater.REMOVE2BYTES
-        self._waitAfterFinish = self._totalProgress > 0
+        self._waitAfterFinish = self._totalSize > 0 or self._numToRemove > 0
 
     def _startDownload(self):
@@ -233 +236 @@
         """Called when the update has been done."""
         gobject.idle_add(self._done)
-        self._restart = self._totalProgress>0
+        self._restart = self._waitAfterFinish
 
     def _done(self):

src/mlx/update.py

@@ -11 +11 @@
 import socket
 import subprocess
+import hashlib
 
 if os.name=="nt":
@@ -18 +19 @@
 
 manifestName = "MLXMANIFEST"
+toremoveName = "toremove"
 
 #------------------------------------------------------------------------------
@@ -43 +45 @@
     def addFile(self, path, size, sum):
         """Add a file to the manifest."""
-        self._files[path] = ( size, sum)
-
+        self._files[path] = (size, sum)
+
+    def addFiles(self, baseDirectory, subdirectory):
+        """Add the files in the given directory and subdirectories of it to the
+        manifest."""
+        directory = baseDirectory
+        for d in subdirectory: directory = os.path.join(directory, d)
+        
+        for entry in os.listdir(directory):
+            fullPath = os.path.join(directory, entry)
+            if os.path.isfile(fullPath):
+                size = os.stat(fullPath).st_size
+                sum = hashlib.md5()
+                with open(fullPath, "rb") as f:
+                    while True:
+                        data = f.read(4096)
+                        if data: sum.update(data)
+                        else: break
+                self.addFile("/".join(subdirectory + [entry]), size,
+                             sum.hexdigest())
+            elif os.path.isdir(fullPath):
+                self.addFiles(baseDirectory, subdirectory + [entry])
+                
     def readFrom(self, file):
         """Read a manifest from the given file object."""
@@ -74 +97 @@
 
         removed = [path for path in self._files if path not in other._files]
-        removed.sort(reverse = True)
         
         return (modifiedAndNew, removed)
@@ -102 +124 @@
         assert False
 
-    def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove):
+    def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
+                     numToRemoveLocal):
         """Called when starting the downloading of the files."""
         self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
-                    str(numToRemove)])
+                    str(numToRemove), str(numToRemoveLocal)])
 
     def setDownloaded(self, downloaded):
@@ -190 +213 @@
 #------------------------------------------------------------------------------
 
+def getToremoveFiles(directory):
+    """Add the files to remove from the toremove directory."""
+    toremove = []
+    toremoveDirectory = os.path.join(directory, toremoveName)
+    if os.path.isdir(toremoveDirectory):
+        for entry in os.listdir(toremoveDirectory):
+            toremove.append(os.path.join(toremoveName, entry))
+    return toremove
+
+#------------------------------------------------------------------------------
+
 def createLocalPath(directory, path):
     """Create a local path from the given manifest path."""
@@ -199 +233 @@
 #------------------------------------------------------------------------------
 
+def getToremoveDir(toremoveDir, directory):
+    """Get the path of the directory that will contain the files that are to be
+    removed."""
+    if toremoveDir is None:
+        toremoveDir = os.path.join(directory, toremoveName)
+        try:
+            os.mkdir(toremoveDir)
+        except:
+            pass
+
+    return toremoveDir
+
+#------------------------------------------------------------------------------
+
+def removeFile(toremoveDir, directory, path):
+    """Remove the file at the given path or store it in a temporary directory.
+
+    If the removal of the file fails, it will be stored in a temporary
+    directory. This is useful for files thay may be open and cannot be removed
+    right away."""
+    try:
+        os.remove(path)
+    except:
+        try:
+            sum = hashlib.md5()
+            sum.update(path)
+            toremoveDir = getToremoveDir(toremoveDir, directory)
+            os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
+        except Exception, e:
+            print "Cannot remove file " + path + ": " + str(e)    
+
+#------------------------------------------------------------------------------
+
 def updateFiles(directory, updateURL, listener,
-                manifest, modifiedAndNew, removed):
+                manifest, modifiedAndNew, removed, localRemoved):
     """Update the files according to the given information."""
     totalSize = 0
@@ -206 +273 @@
         totalSize += size
 
-    listener.setTotalSize(len(modifiedAndNew), totalSize, len(removed))
+    listener.setTotalSize(len(modifiedAndNew), totalSize,
+                          len(removed), len(localRemoved))
 
     downloaded = 0
     fin = None
+    toremoveDir = None
+
     try:        
         updateURL += "/" + os.name
@@ -233 +303 @@
                 fin.close()
                 fin = None
-
+                
         listener.startRenaming()
         count = 0
@@ -241 +311 @@
             downloadedFile = targetFile + "." + sum
             if os.name=="nt" and os.path.isfile(targetFile):
-                try:
-                    os.remove(targetFile)
-                except:
-                    try:
-                        os.remove(targetFile + ".tmp")
-                    except:
-                        pass
-                    os.rename(targetFile, targetFile + ".tmp")
+                removeFile(toremoveDir, directory, targetFile)
             os.rename(downloadedFile, targetFile)
             count += 1
@@ -255 +318 @@
         listener.startRemoving()
         count = 0
+        removed += localRemoved
+        removed.sort(reverse = True)
         for path in removed:            
-            os.remove(path)
-            count += 1
-            pathDirectory = os.path.dirname(path)
+            removePath = createLocalPath(directory, path)
+            removeFile(toremoveDir, directory, removePath)
+
+            removeDirectory = os.path.dirname(removePath)
             try:
-                os.rmdir(pathDirectory)
+                os.removedirs(removeDirectory)
             except:
                 pass
+
+            count += 1
             listener.removed(path, count)
 
@@ -316 +384 @@
             elif command=="setTotalSize":
                 listener.setTotalSize(int(words[1]), long(words[2]),
-                                      int(words[3]))
+                                      int(words[3]), int(words[4]))
             elif command=="setDownloaded":
                 listener.setDownloaded(long(words[1]))
@@ -417 +485 @@
 
     (manifest, updateManifest, modifiedAndNew, removed) = result        
-
-    if not modifiedAndNew and not removed:
+    localRemoved = getToremoveFiles(directory)
+
+    if not modifiedAndNew and not removed and not localRemoved:
         listener.done()
         return
@@ -428 +497 @@
     else:
         updateFiles(directory, updateURL, listener, updateManifest,
-                    modifiedAndNew, removed)
+                    modifiedAndNew, removed, localRemoved)
 
 #------------------------------------------------------------------------------
@@ -458 +527 @@
 
     (modifiedAndNew, removed) = manifest.compare(updateManifest)
+    localRemoved = getToremoveFiles(directory)
 
     updateFiles(directory, config.updateURL,
                 ClientListener(clientSocket),
-                updateManifest, modifiedAndNew, removed)
-
-#------------------------------------------------------------------------------
-
-if __name__ == "__main__":
-    manifest1 = Manifest()
-    manifest1.addFile("file1.exe", 3242, "40398754589435345934")
-    manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
-    manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
-
-    with open("manifest1", "wt") as f:
-        manifest1.writeInto(f)
-
-    manifest2 = Manifest()
-    manifest2.addFile("file1.exe", 4353, "390734659834756349876")
-    manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
-    manifest2.addFile("dir/file4.log", 2390, "56546546546546")
-
-    with open("manifest2", "wt") as f:
-        manifest2.writeInto(f)
-
-    manifest1 = Manifest()
-    with open("manifest1", "rt") as f:
-        manifest1.readFrom(f)
-
-    manifest2 = Manifest()
-    with open("manifest2", "rt") as f:
-        manifest2.readFrom(f)
-
-    (modifiedAndNew, removed) = manifest1.compare(manifest2)
-
-    for (path, size, sum) in modifiedAndNew:
-        print "modified or new:", path, size, sum
-
-    for path in removed:
-        print "removed:", path
-
-#------------------------------------------------------------------------------
+                updateManifest, modifiedAndNew, removed, localRemoved)
+
+#------------------------------------------------------------------------------
+
+def buildManifest(directory):
+    """Build a manifest from the contents of the given directory, into the
+    given directory."""
+
+    manifestPath = os.path.join(directory, manifestName)
+    try:
+        os.remove(manifestPath)
+    except:
+        pass
+    
+    manifest = Manifest()
+    manifest.addFiles(directory, [])
+    with open(manifestPath, "wt") as f:
+        manifest.writeInto(f)
+
+#------------------------------------------------------------------------------
+
+# if __name__ == "__main__":
+#     manifest1 = Manifest()
+#     manifest1.addFile("file1.exe", 3242, "40398754589435345934")
+#     manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
+#     manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
+
+#     with open("manifest1", "wt") as f:
+#         manifest1.writeInto(f)
+
+#     manifest2 = Manifest()
+#     manifest2.addFile("file1.exe", 4353, "390734659834756349876")
+#     manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
+#     manifest2.addFile("dir/file4.log", 2390, "56546546546546")
+
+#     with open("manifest2", "wt") as f:
+#         manifest2.writeInto(f)
+
+#     manifest1 = Manifest()
+#     with open("manifest1", "rt") as f:
+#         manifest1.readFrom(f)
+
+#     manifest2 = Manifest()
+#     with open("manifest2", "rt") as f:
+#         manifest2.readFrom(f)
+
+#     (modifiedAndNew, removed) = manifest1.compare(manifest2)
+
+#     for (path, size, sum) in modifiedAndNew:
+#         print "modified or new:", path, size, sum
+
+#     for path in removed:
+#         print "removed:", path
+
+#------------------------------------------------------------------------------