source: src/mlx/update.py@ 957:aa0c08c6dded

version_0.39_maint
Last change on this file since 957:aa0c08c6dded was 957:aa0c08c6dded, checked in by István Váradi <ivaradi@…>, 5 years ago

More data is read in one call when reading the files to update
File size: 21.8 KB
Line 
1
2from config import Config
3from util import utf2unicode
4
5import os
6import sys
7import urllib2
8import tempfile
9import socket
10import subprocess
11import hashlib
12import traceback
13
14if os.name=="nt":
15 import win32api
16
17#------------------------------------------------------------------------------
18
19## @package mlx.update
20#
21# Automatic update handling.
22#
23# The program can update itself automatically. For this purpose it maintains a
24# manifest of the files installed containing the relative paths, sizes and
25# checksums of each files. When the program starts up, this manifest file is
26# downloaded from the update server and is compared to the local one. Then the
27# updated and new files are downloaded with names that are created by appending
28# the checksum to the actual name, so as not to overwrite any existing files at
29# this stage. If all files are downloaded, the downloaded files are renamed to
30# their real names. On Windows, the old file is removed first to avoid trouble
31# with 'busy' files. If removing a file fails, the file will be moved to a
32# temporary directory, that will be removed when the program starts the next
33# time.
34
35#------------------------------------------------------------------------------
36
37manifestName = "MLXMANIFEST"
38toremoveName = "toremove"
39
40#------------------------------------------------------------------------------
41
42class Manifest(object):
43 """The manifest of the files.
44
45 The manifest file consists of one line for each file. Each line contains 3
46 items separated by tabs:
47 - the path of the file relative to the distribution's root
48 - the size of the file
49 - the MD5 sum of the file."""
50 def __init__(self):
51 """Construct the manifest."""
52 self._files = {}
53
54 @property
55 def files(self):
56 """Get an iterator over the files.
57
58 Each file is returned as a 3-tuple with items as in the file."""
59 for (path, (size, sum)) in self._files.iteritems():
60 yield (path, size, sum)
61
62 def copy(self):
63 """Create a copy of the manifest."""
64 manifest = Manifest()
65 manifest._files = self._files.copy()
66 return manifest
67
68 def addFile(self, path, size, sum):
69 """Add a file to the manifest."""
70 self._files[path] = (size, sum)
71
72 def addFiles(self, baseDirectory, subdirectory):
73 """Add the files in the given directory and subdirectories of it to the
74 manifest."""
75 directory = baseDirectory
76 for d in subdirectory: directory = os.path.join(directory, d)
77
78 for entry in os.listdir(directory):
79 fullPath = os.path.join(directory, entry)
80 if os.path.isfile(fullPath):
81 size = os.stat(fullPath).st_size
82 sum = hashlib.md5()
83 with open(fullPath, "rb") as f:
84 while True:
85 data = f.read(4096)
86 if data: sum.update(data)
87 else: break
88 self.addFile("/".join(subdirectory + [entry]), size,
89 sum.hexdigest())
90 elif os.path.isdir(fullPath):
91 self.addFiles(baseDirectory, subdirectory + [entry])
92
93 def readFrom(self, file):
94 """Read a manifest from the given file object."""
95 for line in iter(file.readline, ""):
96 (path, size, sum) = line.strip().split("\t")
97 self._files[path] = (int(size), sum)
98
99 def writeInto(self, file):
100 """Write the manifest into the file at the given path."""
101 for (path, (size, sum)) in self._files.iteritems():
102 file.write("%s\t%d\t%s\n" % (path, size, sum))
103
104 def compare(self, other):
105 """Compare this manifest with the other one.
106
107 This returns a tuple of two lists:
108 - the files that are either different or are present in other, but not
109 here. Each file is returned as a 3-tuple as with other functions, the
110 size and sum are those of the new file.
111 - the paths of the files that are present here, but not in other."""
112 modifiedAndNew = []
113 for (path, otherSize, otherSum) in other.files:
114 if path in self._files:
115 (size, sum) = self._files[path]
116 if size!=otherSize or sum!=otherSum:
117 modifiedAndNew.append((path, otherSize, otherSum))
118 else:
119 modifiedAndNew.append((path, otherSize, otherSum))
120
121 if os.name=="nt":
122 otherFiles = [path.lower() for path in other._files]
123 else:
124 otherFiles = other._files
125
126 removed = [path for path in self._files if
127 (path.lower() if os.name=="nt" else path) not in otherFiles]
128
129 return (modifiedAndNew, removed)
130
131 def __contains__(self, path):
132 """Determine if the given path is in the manifest."""
133 return path in self._files
134
135 def __getitem__(self, path):
136 """Get data of the file with the given path."""
137 return self._files[path] if path in self._files else None
138
139#------------------------------------------------------------------------------
140
141class ClientListener(object):
142 """A listener that sends any requests via a socket."""
143 def __init__(self, sock):
144 """Construct the listener."""
145 self._sock = sock
146
147 def downloadingManifest(self):
148 """Called when the downloading of the manifest has started."""
149 self._send(["downloadingManifest"])
150
151 def downloadedManifest(self):
152 """Called when the manifest has been downloaded."""
153 self._send(["downloadedManifest"])
154
155 def needSudo(self):
156 """Called when an admin-level program must be called to complete the
157 update.
158
159 This is not valid from a client, as that client is supposed to be the
160 admin-level program :)"""
161 assert False
162
163 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
164 numToRemoveLocal):
165 """Called when starting the downloading of the files."""
166 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
167 str(numToRemove), str(numToRemoveLocal)])
168
169 def setDownloaded(self, downloaded):
170 """Called periodically after downloading a certain amount of data."""
171 self._send(["setDownloaded", str(downloaded)])
172
173 def startRenaming(self):
174 """Called when the renaming of the downloaded files is started."""
175 self._send(["startRenaming"])
176
177 def renamed(self, path, count):
178 """Called when a file has been renamed."""
179 self._send(["renamed", path, str(count)])
180
181 def startRemoving(self):
182 """Called when the removing of files has started."""
183 self._send(["startRemoving"])
184
185 def removed(self, path, count):
186 """Called when a file has been removed."""
187 self._send(["removed", path, str(count)])
188
189 def writingManifest(self):
190 """Called when we have started writing the manifest."""
191 self._send(["writingManifest"])
192
193 def done(self):
194 """Called when the update has completed."""
195 self._send(["done"])
196
197 def failed(self, what):
198 """Called when something has failed."""
199 self._send(["failed", what])
200
201 def _send(self, words):
202 """Send the given words via the socket."""
203 self._sock.send("\t".join(words) + "\n")
204
205#------------------------------------------------------------------------------
206
207def readLocalManifest(directory):
208 """Read the local manifest from the given directory."""
209 manifestPath = os.path.join(directory, manifestName)
210
211 manifest = Manifest()
212 try:
213 with open(manifestPath, "rt") as f:
214 manifest.readFrom(f)
215 except Exception, e:
216 print "Error reading the manifest, ignoring:", utf2unicode(str(e))
217 manifest = Manifest()
218
219 return manifest
220
221#------------------------------------------------------------------------------
222
223def prepareUpdate(directory, updateURL, listener):
224 """Prepare the update by downloading the manifest and comparing it with the
225 local one."""
226 manifest = readLocalManifest(directory)
227
228 updateURL += "/" + os.name
229
230 listener.downloadingManifest()
231 f = None
232 try:
233 updateManifest = Manifest()
234 f= urllib2.urlopen(updateURL + "/" + manifestName)
235 updateManifest.readFrom(f)
236
237 except Exception, e:
238 error = utf2unicode(str(e))
239 print >> sys.stderr, "Error downloading manifest:", error
240 listener.failed(error)
241 return None
242 finally:
243 if f is not None: f.close()
244
245 listener.downloadedManifest()
246
247 (modifiedAndNew, removed) = manifest.compare(updateManifest)
248
249 return (manifest, updateManifest, modifiedAndNew, removed)
250
251#------------------------------------------------------------------------------
252
253def getToremoveFiles(directory):
254 """Add the files to remove from the toremove directory."""
255 toremove = []
256 toremoveDirectory = os.path.join(directory, toremoveName)
257 if os.path.isdir(toremoveDirectory):
258 for entry in os.listdir(toremoveDirectory):
259 toremove.append(os.path.join(toremoveName, entry))
260 return toremove
261
262#------------------------------------------------------------------------------
263
264def createLocalPath(directory, path):
265 """Create a local path from the given manifest path."""
266 localPath = directory
267 for element in path.split("/"):
268 localPath = os.path.join(localPath, element)
269 return localPath
270
271#------------------------------------------------------------------------------
272
273def getToremoveDir(toremoveDir, directory):
274 """Get the path of the directory that will contain the files that are to be
275 removed."""
276 if toremoveDir is None:
277 toremoveDir = os.path.join(directory, toremoveName)
278 try:
279 os.mkdir(toremoveDir)
280 except:
281 pass
282
283 return toremoveDir
284
285#------------------------------------------------------------------------------
286
287def removeFile(toremoveDir, directory, path):
288 """Remove the file at the given path or store it in a temporary directory.
289
290 If the removal of the file fails, it will be stored in a temporary
291 directory. This is useful for files thay may be open and cannot be removed
292 right away."""
293 try:
294 os.remove(path)
295 except:
296 try:
297 sum = hashlib.md5()
298 sum.update(path)
299 toremoveDir = getToremoveDir(toremoveDir, directory)
300 targetPath = os.path.join(toremoveDir, sum.hexdigest())
301 try:
302 os.remove(targetPath)
303 except:
304 pass
305 os.rename(path, targetPath)
306 except Exception, e:
307 print "Cannot remove file " + path + ": " + utf2unicode(str(e))
308
309#------------------------------------------------------------------------------
310
311def removeFiles(directory, listener, removed, count):
312 """Remove the given files."""
313 toremoveDir = None
314
315 listener.startRemoving()
316
317 removed.sort(reverse = True)
318 for path in removed:
319 removePath = createLocalPath(directory, path)
320 removeFile(toremoveDir, directory, removePath)
321
322 removeDirectory = os.path.dirname(removePath)
323 try:
324 os.removedirs(removeDirectory)
325 except:
326 pass
327
328 count += 1
329 listener.removed(path, count)
330
331 return count
332
333#------------------------------------------------------------------------------
334
335def updateFiles(directory, updateURL, listener,
336 manifest, modifiedAndNew, removed, localRemoved):
337 """Update the files according to the given information."""
338 totalSize = 0
339 for (path, size, sum) in modifiedAndNew:
340 totalSize += size
341
342 listener.setTotalSize(len(modifiedAndNew), totalSize,
343 len(removed), len(localRemoved))
344
345 downloaded = 0
346 fin = None
347 toremoveDir = None
348
349 try:
350 updateURL += "/" + os.name
351
352 removeCount = 0
353 if localRemoved:
354 removeCount = removeFiles(directory, listener,
355 localRemoved, removeCount)
356
357 for (path, size, sum) in modifiedAndNew:
358 targetFile = createLocalPath(directory, path)
359 targetFile += "."
360 targetFile += sum
361
362 targetDirectory = os.path.dirname(targetFile)
363 if targetDirectory and not os.path.isdir(targetDirectory):
364 os.makedirs(targetDirectory)
365
366 with open(targetFile, "wb") as fout:
367 fin = urllib2.urlopen(updateURL + "/" + path)
368 while True:
369 data = fin.read(256*1024)
370 if not data:
371 break
372 fout.write(data)
373 downloaded += len(data)
374 listener.setDownloaded(downloaded)
375 fin.close()
376 fin = None
377
378 listener.startRenaming()
379 count = 0
380 for (path, size, sum) in modifiedAndNew:
381 targetFile = createLocalPath(directory, path)
382
383 downloadedFile = targetFile + "." + sum
384 if os.name=="nt" and os.path.isfile(targetFile):
385 removeFile(toremoveDir, directory, targetFile)
386 os.rename(downloadedFile, targetFile)
387 count += 1
388 listener.renamed(path, count)
389
390 removeFiles(directory, listener, removed, removeCount)
391
392 listener.writingManifest()
393
394 manifestPath = os.path.join(directory, manifestName)
395 with open(manifestPath, "wt") as f:
396 manifest.writeInto(f)
397
398 listener.done()
399 except Exception, e:
400 exc = traceback.format_exc()
401 print >> sys.stderr, utf2unicode(exc)
402
403 error = utf2unicode(str(e))
404 print >> sys.stderr, "Error:", error
405
406 listener.failed(error)
407
408#------------------------------------------------------------------------------
409
410def isDirectoryWritable(directory):
411 """Determine if the given directory can be written."""
412 checkFile = os.path.join(directory, "writable.chk")
413 try:
414 f = open(checkFile, "wt")
415 f.close()
416 return True
417 except Exception, e:
418 return False
419 finally:
420 try:
421 os.remove(checkFile)
422 except:
423 pass
424
425#------------------------------------------------------------------------------
426
427def processMLXUpdate(buffer, listener):
428 """Process the given buffer supposedly containing a list of commands."""
429 endsWithLine = buffer[-1]=="\n"
430 lines = buffer.splitlines()
431
432 if endsWithLine:
433 buffer = ""
434 else:
435 buffer = lines[-1]
436 lines = lines[:-1]
437
438 for line in lines:
439 words = line.split("\t")
440 try:
441 command = words[0]
442 if command=="downloadingManifest":
443 listener.downloadingManifest()
444 elif command=="downloadedManifest":
445 listener.downloadedManifest()
446 elif command=="setTotalSize":
447 listener.setTotalSize(int(words[1]), long(words[2]),
448 int(words[3]), int(words[4]))
449 elif command=="setDownloaded":
450 listener.setDownloaded(long(words[1]))
451 elif command=="startRenaming":
452 listener.startRenaming()
453 elif command=="renamed":
454 listener.renamed(words[1], int(words[2]))
455 elif command=="startRemoving":
456 listener.startRemoving()
457 elif command=="removed":
458 listener.removed(words[1], int(words[2]))
459 elif command=="writingManifest":
460 listener.writingManifest()
461 elif command=="done":
462 listener.done()
463 elif command=="failed":
464 listener.failed(words[1])
465 except Exception, e:
466 print >> sys.stderr, "Failed to parse line '%s': %s" % \
467 (line, utf2unicode(str(e)))
468
469 return buffer
470
471#------------------------------------------------------------------------------
472
473def sudoUpdate(directory, updateURL, listener, manifest):
474 """Perform the update via the mlxupdate program."""
475 manifestFD = None
476 manifestFile = None
477 serverSocket = None
478 mlxUpdateSocket = None
479 try:
480 (manifestFD, manifestFile) = tempfile.mkstemp()
481 f = os.fdopen(manifestFD, "wt")
482 try:
483 manifest.writeInto(f)
484 finally:
485 f.close()
486 manifestFD = None
487
488 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
489 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
490 serverSocket.bind(("127.0.0.1", 0))
491 (_host, port) = serverSocket.getsockname()
492 serverSocket.listen(1)
493
494
495 if os.name=="nt":
496 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
497 str(port) + " " + manifestFile + " " + updateURL, "", 1)
498 else:
499 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
500 str(port), manifestFile, updateURL],
501 shell = os.name=="nt")
502
503 (mlxUpdateSocket, _) = serverSocket.accept()
504 serverSocket.close()
505 serverSocket = None
506
507 buffer = ""
508 while True:
509 data = mlxUpdateSocket.recv(4096)
510 if not data:
511 break;
512
513 buffer += data
514 buffer = processMLXUpdate(buffer, listener)
515
516 mlxUpdateSocket.close()
517 mlxUpdateSocket = None
518
519 if os.name!="nt":
520 process.wait()
521
522 except Exception, e:
523 error = utf2unicode(str(e))
524 print >> sys.stderr, "Failed updating:", error
525 listener.failed(error)
526 finally:
527 if serverSocket is not None:
528 try:
529 serverSocket.close()
530 except:
531 pass
532 if mlxUpdateSocket is not None:
533 try:
534 mlxUpdateSocket.close()
535 except:
536 pass
537 if manifestFD is not None:
538 try:
539 os.close(manifestFD)
540 except:
541 pass
542 if manifestFile is not None:
543 try:
544 os.remove(manifestFile)
545 except:
546 pass
547
548
549#------------------------------------------------------------------------------
550
551def update(directory, updateURL, listener, fromGUI = False):
552 """Perform the update."""
553 try:
554 result = prepareUpdate(directory, updateURL, listener)
555 if result is None:
556 return
557
558 (manifest, updateManifest, modifiedAndNew, removed) = result
559 localRemoved = getToremoveFiles(directory)
560
561 if not modifiedAndNew and not removed and not localRemoved:
562 listener.done()
563 return
564
565 if fromGUI and not isDirectoryWritable(directory):
566 if listener.needSudo():
567 sudoUpdate(directory, updateURL, listener, updateManifest)
568 else:
569 updateFiles(directory, updateURL, listener, updateManifest,
570 modifiedAndNew, removed, localRemoved)
571 except Exception, e:
572 exc = traceback.format_exc()
573 print >> sys.stderr, utf2unicode(exc)
574
575 error = utf2unicode(str(e))
576 print >> sys.stderr, "Update error:", error
577
578 listener.failed(error)
579
580#------------------------------------------------------------------------------
581
582def updateProcess():
583 """This is called in the child process, when we need a child process."""
584 try:
585 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
586 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
587
588 directory = os.path.dirname(sys.argv[0])
589
590 manifest = readLocalManifest(directory)
591
592 updateManifest = Manifest()
593 with open(sys.argv[2], "rt") as f:
594 updateManifest.readFrom(f)
595
596 (modifiedAndNew, removed) = manifest.compare(updateManifest)
597 localRemoved = getToremoveFiles(directory)
598
599 updateFiles(directory, sys.argv[3],
600 ClientListener(clientSocket),
601 updateManifest, modifiedAndNew, removed, localRemoved)
602 except:
603 exc = traceback.format_exc()
604 print >> sys.stderr, utf2unicode(exc)
605
606#------------------------------------------------------------------------------
607
608def buildManifest(directory):
609 """Build a manifest from the contents of the given directory, into the
610 given directory."""
611
612 manifestPath = os.path.join(directory, manifestName)
613 try:
614 os.remove(manifestPath)
615 except:
616 pass
617
618 manifest = Manifest()
619 manifest.addFiles(directory, [])
620 with open(manifestPath, "wt") as f:
621 manifest.writeInto(f)
622
623#------------------------------------------------------------------------------
624
625# if __name__ == "__main__":
626# manifest1 = Manifest()
627# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
628# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
629# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
630
631# with open("manifest1", "wt") as f:
632# manifest1.writeInto(f)
633
634# manifest2 = Manifest()
635# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
636# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
637# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
638
639# with open("manifest2", "wt") as f:
640# manifest2.writeInto(f)
641
642# manifest1 = Manifest()
643# with open("manifest1", "rt") as f:
644# manifest1.readFrom(f)
645
646# manifest2 = Manifest()
647# with open("manifest2", "rt") as f:
648# manifest2.readFrom(f)
649
650# (modifiedAndNew, removed) = manifest1.compare(manifest2)
651
652# for (path, size, sum) in modifiedAndNew:
653# print "modified or new:", path, size, sum
654
655# for path in removed:
656# print "removed:", path
657
658#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.