source: src/mlx/update.py@ 787:7fa0ff7749c5

Last change on this file since 787:7fa0ff7749c5 was 781:d4455b2cd469, checked in by István Váradi <ivaradi@…>, 8 years ago

The files already removable are removed in advance (re #298)
File size: 20.9 KB
Line 
1
2from config import Config
3from util import utf2unicode
4
5import os
6import sys
7import urllib2
8import tempfile
9import socket
10import subprocess
11import hashlib
12
13if os.name=="nt":
14 import win32api
15
16#------------------------------------------------------------------------------
17
18## @package mlx.update
19#
20# Automatic update handling.
21#
22# The program can update itself automatically. For this purpose it maintains a
23# manifest of the files installed containing the relative paths, sizes and
24# checksums of each files. When the program starts up, this manifest file is
25# downloaded from the update server and is compared to the local one. Then the
26# updated and new files are downloaded with names that are created by appending
27# the checksum to the actual name, so as not to overwrite any existing files at
28# this stage. If all files are downloaded, the downloaded files are renamed to
29# their real names. On Windows, the old file is removed first to avoid trouble
30# with 'busy' files. If removing a file fails, the file will be moved to a
31# temporary directory, that will be removed when the program starts the next
32# time.
33
34#------------------------------------------------------------------------------
35
36manifestName = "MLXMANIFEST"
37toremoveName = "toremove"
38
39#------------------------------------------------------------------------------
40
41class Manifest(object):
42 """The manifest of the files.
43
44 The manifest file consists of one line for each file. Each line contains 3
45 items separated by tabs:
46 - the path of the file relative to the distribution's root
47 - the size of the file
48 - the MD5 sum of the file."""
49 def __init__(self):
50 """Construct the manifest."""
51 self._files = {}
52
53 @property
54 def files(self):
55 """Get an iterator over the files.
56
57 Each file is returned as a 3-tuple with items as in the file."""
58 for (path, (size, sum)) in self._files.iteritems():
59 yield (path, size, sum)
60
61 def copy(self):
62 """Create a copy of the manifest."""
63 manifest = Manifest()
64 manifest._files = self._files.copy()
65 return manifest
66
67 def addFile(self, path, size, sum):
68 """Add a file to the manifest."""
69 self._files[path] = (size, sum)
70
71 def addFiles(self, baseDirectory, subdirectory):
72 """Add the files in the given directory and subdirectories of it to the
73 manifest."""
74 directory = baseDirectory
75 for d in subdirectory: directory = os.path.join(directory, d)
76
77 for entry in os.listdir(directory):
78 fullPath = os.path.join(directory, entry)
79 if os.path.isfile(fullPath):
80 size = os.stat(fullPath).st_size
81 sum = hashlib.md5()
82 with open(fullPath, "rb") as f:
83 while True:
84 data = f.read(4096)
85 if data: sum.update(data)
86 else: break
87 self.addFile("/".join(subdirectory + [entry]), size,
88 sum.hexdigest())
89 elif os.path.isdir(fullPath):
90 self.addFiles(baseDirectory, subdirectory + [entry])
91
92 def readFrom(self, file):
93 """Read a manifest from the given file object."""
94 for line in iter(file.readline, ""):
95 (path, size, sum) = line.strip().split("\t")
96 self._files[path] = (int(size), sum)
97
98 def writeInto(self, file):
99 """Write the manifest into the file at the given path."""
100 for (path, (size, sum)) in self._files.iteritems():
101 file.write("%s\t%d\t%s\n" % (path, size, sum))
102
103 def compare(self, other):
104 """Compare this manifest with the other one.
105
106 This returns a tuple of two lists:
107 - the files that are either different or are present in other, but not
108 here. Each file is returned as a 3-tuple as with other functions, the
109 size and sum are those of the new file.
110 - the paths of the files that are present here, but not in other."""
111 modifiedAndNew = []
112 for (path, otherSize, otherSum) in other.files:
113 if path in self._files:
114 (size, sum) = self._files[path]
115 if size!=otherSize or sum!=otherSum:
116 modifiedAndNew.append((path, otherSize, otherSum))
117 else:
118 modifiedAndNew.append((path, otherSize, otherSum))
119
120 removed = [path for path in self._files if path not in other._files]
121
122 return (modifiedAndNew, removed)
123
124 def __contains__(self, path):
125 """Determine if the given path is in the manifest."""
126 return path in self._files
127
128 def __getitem__(self, path):
129 """Get data of the file with the given path."""
130 return self._files[path] if path in self._files else None
131
132#------------------------------------------------------------------------------
133
134class ClientListener(object):
135 """A listener that sends any requests via a socket."""
136 def __init__(self, sock):
137 """Construct the listener."""
138 self._sock = sock
139
140 def downloadingManifest(self):
141 """Called when the downloading of the manifest has started."""
142 self._send(["downloadingManifest"])
143
144 def downloadedManifest(self):
145 """Called when the manifest has been downloaded."""
146 self._send(["downloadedManifest"])
147
148 def needSudo(self):
149 """Called when an admin-level program must be called to complete the
150 update.
151
152 This is not valid from a client, as that client is supposed to be the
153 admin-level program :)"""
154 assert False
155
156 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
157 numToRemoveLocal):
158 """Called when starting the downloading of the files."""
159 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
160 str(numToRemove), str(numToRemoveLocal)])
161
162 def setDownloaded(self, downloaded):
163 """Called periodically after downloading a certain amount of data."""
164 self._send(["setDownloaded", str(downloaded)])
165
166 def startRenaming(self):
167 """Called when the renaming of the downloaded files is started."""
168 self._send(["startRenaming"])
169
170 def renamed(self, path, count):
171 """Called when a file has been renamed."""
172 self._send(["renamed", path, str(count)])
173
174 def startRemoving(self):
175 """Called when the removing of files has started."""
176 self._send(["startRemoving"])
177
178 def removed(self, path, count):
179 """Called when a file has been removed."""
180 self._send(["removed", path, str(count)])
181
182 def writingManifest(self):
183 """Called when we have started writing the manifest."""
184 self._send(["writingManifest"])
185
186 def done(self):
187 """Called when the update has completed."""
188 self._send(["done"])
189
190 def failed(self, what):
191 """Called when something has failed."""
192 self._send(["failed", what])
193
194 def _send(self, words):
195 """Send the given words via the socket."""
196 self._sock.send("\t".join(words) + "\n")
197
198#------------------------------------------------------------------------------
199
200def readLocalManifest(directory):
201 """Read the local manifest from the given directory."""
202 manifestPath = os.path.join(directory, manifestName)
203
204 manifest = Manifest()
205 try:
206 with open(manifestPath, "rt") as f:
207 manifest.readFrom(f)
208 except Exception, e:
209 print "Error reading the manifest, ignoring:", utf2unicode(str(e))
210 manifest = Manifest()
211
212 return manifest
213
214#------------------------------------------------------------------------------
215
216def prepareUpdate(directory, updateURL, listener):
217 """Prepare the update by downloading the manifest and comparing it with the
218 local one."""
219 manifest = readLocalManifest(directory)
220
221 updateURL += "/" + os.name
222
223 listener.downloadingManifest()
224 f = None
225 try:
226 updateManifest = Manifest()
227 f= urllib2.urlopen(updateURL + "/" + manifestName)
228 updateManifest.readFrom(f)
229
230 except Exception, e:
231 error = utf2unicode(str(e))
232 print >> sys.stderr, "Error downloading manifest:", error
233 listener.failed(error)
234 return None
235 finally:
236 if f is not None: f.close()
237
238 listener.downloadedManifest()
239
240 (modifiedAndNew, removed) = manifest.compare(updateManifest)
241
242 return (manifest, updateManifest, modifiedAndNew, removed)
243
244#------------------------------------------------------------------------------
245
246def getToremoveFiles(directory):
247 """Add the files to remove from the toremove directory."""
248 toremove = []
249 toremoveDirectory = os.path.join(directory, toremoveName)
250 if os.path.isdir(toremoveDirectory):
251 for entry in os.listdir(toremoveDirectory):
252 toremove.append(os.path.join(toremoveName, entry))
253 return toremove
254
255#------------------------------------------------------------------------------
256
257def createLocalPath(directory, path):
258 """Create a local path from the given manifest path."""
259 localPath = directory
260 for element in path.split("/"):
261 localPath = os.path.join(localPath, element)
262 return localPath
263
264#------------------------------------------------------------------------------
265
266def getToremoveDir(toremoveDir, directory):
267 """Get the path of the directory that will contain the files that are to be
268 removed."""
269 if toremoveDir is None:
270 toremoveDir = os.path.join(directory, toremoveName)
271 try:
272 os.mkdir(toremoveDir)
273 except:
274 pass
275
276 return toremoveDir
277
278#------------------------------------------------------------------------------
279
280def removeFile(toremoveDir, directory, path):
281 """Remove the file at the given path or store it in a temporary directory.
282
283 If the removal of the file fails, it will be stored in a temporary
284 directory. This is useful for files thay may be open and cannot be removed
285 right away."""
286 try:
287 os.remove(path)
288 except:
289 try:
290 sum = hashlib.md5()
291 sum.update(path)
292 toremoveDir = getToremoveDir(toremoveDir, directory)
293 os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
294 except Exception, e:
295 print "Cannot remove file " + path + ": " + utf2unicode(str(e))
296
297#------------------------------------------------------------------------------
298
299def removeFiles(directory, listener, removed, count):
300 """Remove the given files."""
301 toremoveDir = None
302
303 listener.startRemoving()
304
305 removed.sort(reverse = True)
306 for path in removed:
307 removePath = createLocalPath(directory, path)
308 removeFile(toremoveDir, directory, removePath)
309
310 removeDirectory = os.path.dirname(removePath)
311 try:
312 os.removedirs(removeDirectory)
313 except:
314 pass
315
316 count += 1
317 listener.removed(path, count)
318
319 return count
320
321#------------------------------------------------------------------------------
322
323def updateFiles(directory, updateURL, listener,
324 manifest, modifiedAndNew, removed, localRemoved):
325 """Update the files according to the given information."""
326 totalSize = 0
327 for (path, size, sum) in modifiedAndNew:
328 totalSize += size
329
330 listener.setTotalSize(len(modifiedAndNew), totalSize,
331 len(removed), len(localRemoved))
332
333 downloaded = 0
334 fin = None
335 toremoveDir = None
336
337 try:
338 updateURL += "/" + os.name
339
340 removeCount = 0
341 if localRemoved:
342 removeCount = removeFiles(directory, listener,
343 localRemoved, removeCount)
344
345 for (path, size, sum) in modifiedAndNew:
346 targetFile = createLocalPath(directory, path)
347 targetFile += "."
348 targetFile += sum
349
350 targetDirectory = os.path.dirname(targetFile)
351 if not os.path.isdir(targetDirectory):
352 os.makedirs(targetDirectory)
353
354 with open(targetFile, "wb") as fout:
355 fin = urllib2.urlopen(updateURL + "/" + path)
356 while True:
357 data = fin.read(4096)
358 if not data:
359 break
360 fout.write(data)
361 downloaded += len(data)
362 listener.setDownloaded(downloaded)
363 fin.close()
364 fin = None
365
366 listener.startRenaming()
367 count = 0
368 for (path, size, sum) in modifiedAndNew:
369 targetFile = createLocalPath(directory, path)
370
371 downloadedFile = targetFile + "." + sum
372 if os.name=="nt" and os.path.isfile(targetFile):
373 removeFile(toremoveDir, directory, targetFile)
374 os.rename(downloadedFile, targetFile)
375 count += 1
376 listener.renamed(path, count)
377
378 removeFiles(directory, listener, removed, removeCount)
379
380 listener.writingManifest()
381
382 manifestPath = os.path.join(directory, manifestName)
383 with open(manifestPath, "wt") as f:
384 manifest.writeInto(f)
385
386 listener.done()
387 except Exception, e:
388 error = utf2unicode(str(e))
389 print >> sys.stderr, "Error:", error
390 listener.failed(error)
391
392#------------------------------------------------------------------------------
393
394def isDirectoryWritable(directory):
395 """Determine if the given directory can be written."""
396 checkFile = os.path.join(directory, "writable.chk")
397 try:
398 f = open(checkFile, "wt")
399 f.close()
400 return True
401 except Exception, e:
402 return False
403 finally:
404 try:
405 os.remove(checkFile)
406 except:
407 pass
408
409#------------------------------------------------------------------------------
410
411def processMLXUpdate(buffer, listener):
412 """Process the given buffer supposedly containing a list of commands."""
413 endsWithLine = buffer[-1]=="\n"
414 lines = buffer.splitlines()
415
416 if endsWithLine:
417 buffer = ""
418 else:
419 buffer = lines[-1]
420 lines = lines[:-1]
421
422 for line in lines:
423 words = line.split("\t")
424 try:
425 command = words[0]
426 if command=="downloadingManifest":
427 listener.downloadingManifest()
428 elif command=="downloadedManifest":
429 listener.downloadedManifest()
430 elif command=="setTotalSize":
431 listener.setTotalSize(int(words[1]), long(words[2]),
432 int(words[3]), int(words[4]))
433 elif command=="setDownloaded":
434 listener.setDownloaded(long(words[1]))
435 elif command=="startRenaming":
436 listener.startRenaming()
437 elif command=="renamed":
438 listener.renamed(words[1], int(words[2]))
439 elif command=="startRemoving":
440 listener.startRemoving()
441 elif command=="removed":
442 listener.removed(words[1], int(words[2]))
443 elif command=="writingManifest":
444 listener.writingManifest()
445 elif command=="done":
446 listener.done()
447 elif command=="failed":
448 listener.failed(words[1])
449 except Exception, e:
450 print >> sys.stderr, "Failed to parse line '%s': %s" % \
451 (line, utf2unicode(str(e)))
452
453 return buffer
454
455#------------------------------------------------------------------------------
456
457def sudoUpdate(directory, updateURL, listener, manifest):
458 """Perform the update via the mlxupdate program."""
459 manifestFD = None
460 manifestFile = None
461 serverSocket = None
462 mlxUpdateSocket = None
463 try:
464 (manifestFD, manifestFile) = tempfile.mkstemp()
465 f = os.fdopen(manifestFD, "wt")
466 try:
467 manifest.writeInto(f)
468 finally:
469 f.close()
470 manifestFD = None
471
472 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
473 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
474 serverSocket.bind(("127.0.0.1", 0))
475 (_host, port) = serverSocket.getsockname()
476 serverSocket.listen(1)
477
478
479 if os.name=="nt":
480 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
481 str(port) + " " + manifestFile + " " + updateURL, "", 1)
482 else:
483 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
484 str(port), manifestFile, updateURL],
485 shell = os.name=="nt")
486
487 (mlxUpdateSocket, _) = serverSocket.accept()
488 serverSocket.close()
489 serverSocket = None
490
491 buffer = ""
492 while True:
493 data = mlxUpdateSocket.recv(4096)
494 if not data:
495 break;
496
497 buffer += data
498 buffer = processMLXUpdate(buffer, listener)
499
500 mlxUpdateSocket.close()
501 mlxUpdateSocket = None
502
503 if os.name!="nt":
504 process.wait()
505
506 except Exception, e:
507 error = utf2unicode(str(e))
508 print >> sys.stderr, "Failed updating:", error
509 listener.failed(error)
510 finally:
511 if serverSocket is not None:
512 try:
513 serverSocket.close()
514 except:
515 pass
516 if mlxUpdateSocket is not None:
517 try:
518 mlxUpdateSocket.close()
519 except:
520 pass
521 if manifestFD is not None:
522 try:
523 os.close(manifestFD)
524 except:
525 pass
526 if manifestFile is not None:
527 try:
528 os.remove(manifestFile)
529 except:
530 pass
531
532
533#------------------------------------------------------------------------------
534
535def update(directory, updateURL, listener, fromGUI = False):
536 """Perform the update."""
537 result = prepareUpdate(directory, updateURL, listener)
538 if result is None:
539 return
540
541 (manifest, updateManifest, modifiedAndNew, removed) = result
542 localRemoved = getToremoveFiles(directory)
543
544 if not modifiedAndNew and not removed and not localRemoved:
545 listener.done()
546 return
547
548 if fromGUI and not isDirectoryWritable(directory):
549 if listener.needSudo():
550 sudoUpdate(directory, updateURL, listener, updateManifest)
551 else:
552 updateFiles(directory, updateURL, listener, updateManifest,
553 modifiedAndNew, removed, localRemoved)
554
555#------------------------------------------------------------------------------
556
557def updateProcess():
558 """This is called in the child process, when we need a child process."""
559 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
560 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
561
562 directory = os.path.dirname(sys.argv[0])
563
564 manifest = readLocalManifest(directory)
565
566 updateManifest = Manifest()
567 with open(sys.argv[2], "rt") as f:
568 updateManifest.readFrom(f)
569
570 (modifiedAndNew, removed) = manifest.compare(updateManifest)
571 localRemoved = getToremoveFiles(directory)
572
573 updateFiles(directory, sys.argv[3],
574 ClientListener(clientSocket),
575 updateManifest, modifiedAndNew, removed, localRemoved)
576
577#------------------------------------------------------------------------------
578
579def buildManifest(directory):
580 """Build a manifest from the contents of the given directory, into the
581 given directory."""
582
583 manifestPath = os.path.join(directory, manifestName)
584 try:
585 os.remove(manifestPath)
586 except:
587 pass
588
589 manifest = Manifest()
590 manifest.addFiles(directory, [])
591 with open(manifestPath, "wt") as f:
592 manifest.writeInto(f)
593
594#------------------------------------------------------------------------------
595
596# if __name__ == "__main__":
597# manifest1 = Manifest()
598# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
599# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
600# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
601
602# with open("manifest1", "wt") as f:
603# manifest1.writeInto(f)
604
605# manifest2 = Manifest()
606# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
607# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
608# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
609
610# with open("manifest2", "wt") as f:
611# manifest2.writeInto(f)
612
613# manifest1 = Manifest()
614# with open("manifest1", "rt") as f:
615# manifest1.readFrom(f)
616
617# manifest2 = Manifest()
618# with open("manifest2", "rt") as f:
619# manifest2.readFrom(f)
620
621# (modifiedAndNew, removed) = manifest1.compare(manifest2)
622
623# for (path, size, sum) in modifiedAndNew:
624# print "modified or new:", path, size, sum
625
626# for path in removed:
627# print "removed:", path
628
629#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.