source: src/mlx/update.py@ 166:e4ba22b7a13b

Last change on this file since 166:e4ba22b7a13b was 129:dcca616a78ca, checked in by István Váradi <ivaradi@…>, 12 years ago

Fixed the external updater process to load the configuration so that the URL is correct
File size: 19.2 KB
Line 
1# Module handling the download of updates
2
3#------------------------------------------------------------------------------
4
5from config import Config
6
7import os
8import sys
9import urllib2
10import tempfile
11import socket
12import subprocess
13import hashlib
14
15if os.name=="nt":
16 import win32api
17
18#------------------------------------------------------------------------------
19
20manifestName = "MLXMANIFEST"
21toremoveName = "toremove"
22
23#------------------------------------------------------------------------------
24
25class Manifest(object):
26 """The manifest of the files.
27
28 The manifest file consists of one line for each file. Each line contains 3
29 items separated by tabs:
30 - the path of the file relative to the distribution's root
31 - the size of the file
32 - the MD5 sum of the file."""
33 def __init__(self):
34 """Construct the manifest."""
35 self._files = {}
36
37 @property
38 def files(self):
39 """Get an iterator over the files.
40
41 Each file is returned as a 3-tuple with items as in the file."""
42 for (path, (size, sum)) in self._files.iteritems():
43 yield (path, size, sum)
44
45 def addFile(self, path, size, sum):
46 """Add a file to the manifest."""
47 self._files[path] = (size, sum)
48
49 def addFiles(self, baseDirectory, subdirectory):
50 """Add the files in the given directory and subdirectories of it to the
51 manifest."""
52 directory = baseDirectory
53 for d in subdirectory: directory = os.path.join(directory, d)
54
55 for entry in os.listdir(directory):
56 fullPath = os.path.join(directory, entry)
57 if os.path.isfile(fullPath):
58 size = os.stat(fullPath).st_size
59 sum = hashlib.md5()
60 with open(fullPath, "rb") as f:
61 while True:
62 data = f.read(4096)
63 if data: sum.update(data)
64 else: break
65 self.addFile("/".join(subdirectory + [entry]), size,
66 sum.hexdigest())
67 elif os.path.isdir(fullPath):
68 self.addFiles(baseDirectory, subdirectory + [entry])
69
70 def readFrom(self, file):
71 """Read a manifest from the given file object."""
72 for line in iter(file.readline, ""):
73 (path, size, sum) = line.strip().split("\t")
74 self._files[path] = (int(size), sum)
75
76 def writeInto(self, file):
77 """Write the manifest into the file at the given path."""
78 for (path, (size, sum)) in self._files.iteritems():
79 file.write("%s\t%d\t%s\n" % (path, size, sum))
80
81 def compare(self, other):
82 """Compare this manifest with the other one.
83
84 This returns a tuple of two lists:
85 - the files that are either different or are present in other, but not
86 here. Each file is returned as a 3-tuple as with other functions, the
87 size and sum are those of the new file.
88 - the paths of the files that are present here, but not in other."""
89 modifiedAndNew = []
90 for (path, otherSize, otherSum) in other.files:
91 if path in self._files:
92 (size, sum) = self._files[path]
93 if size!=otherSize or sum!=otherSum:
94 modifiedAndNew.append((path, otherSize, otherSum))
95 else:
96 modifiedAndNew.append((path, otherSize, otherSum))
97
98 removed = [path for path in self._files if path not in other._files]
99
100 return (modifiedAndNew, removed)
101
102#------------------------------------------------------------------------------
103
104class ClientListener(object):
105 """A listener that sends any requests via a socket."""
106 def __init__(self, sock):
107 """Construct the listener."""
108 self._sock = sock
109
110 def downloadingManifest(self):
111 """Called when the downloading of the manifest has started."""
112 self._send(["downloadingManifest"])
113
114 def downloadedManifest(self):
115 """Called when the manifest has been downloaded."""
116 self._send(["downloadedManifest"])
117
118 def needSudo(self):
119 """Called when an admin-level program must be called to complete the
120 update.
121
122 This is not valid from a client, as that client is supposed to be the
123 admin-level program :)"""
124 assert False
125
126 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
127 numToRemoveLocal):
128 """Called when starting the downloading of the files."""
129 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
130 str(numToRemove), str(numToRemoveLocal)])
131
132 def setDownloaded(self, downloaded):
133 """Called periodically after downloading a certain amount of data."""
134 self._send(["setDownloaded", str(downloaded)])
135
136 def startRenaming(self):
137 """Called when the renaming of the downloaded files is started."""
138 self._send(["startRenaming"])
139
140 def renamed(self, path, count):
141 """Called when a file has been renamed."""
142 self._send(["renamed", path, str(count)])
143
144 def startRemoving(self):
145 """Called when the removing of files has started."""
146 self._send(["startRemoving"])
147
148 def removed(self, path, count):
149 """Called when a file has been removed."""
150 self._send(["removed", path, str(count)])
151
152 def writingManifest(self):
153 """Called when we have started writing the manifest."""
154 self._send(["writingManifest"])
155
156 def done(self):
157 """Called when the update has completed."""
158 self._send(["done"])
159
160 def failed(self, what):
161 """Called when something has failed."""
162 self._send(["failed", what])
163
164 def _send(self, words):
165 """Send the given words via the socket."""
166 self._sock.send("\t".join(words) + "\n")
167
168#------------------------------------------------------------------------------
169
170def readLocalManifest(directory):
171 """Read the local manifest from the given directory."""
172 manifestPath = os.path.join(directory, manifestName)
173
174 manifest = Manifest()
175 try:
176 with open(manifestPath, "rt") as f:
177 manifest.readFrom(f)
178 except Exception, e:
179 print "Error reading the manifest, ignoring:", str(e)
180 manifest = Manifest()
181
182 return manifest
183
184#------------------------------------------------------------------------------
185
186def prepareUpdate(directory, updateURL, listener):
187 """Prepare the update by downloading the manifest and comparing it with the
188 local one."""
189 manifest = readLocalManifest(directory)
190
191 updateURL += "/" + os.name
192
193 listener.downloadingManifest()
194 f = None
195 try:
196 updateManifest = Manifest()
197 f= urllib2.urlopen(updateURL + "/" + manifestName)
198 updateManifest.readFrom(f)
199
200 except Exception, e:
201 print >> sys.stderr, "Error downloading manifest:", str(e)
202 listener.failed(str(e))
203 return None
204 finally:
205 if f is not None: f.close()
206
207 listener.downloadedManifest()
208
209 (modifiedAndNew, removed) = manifest.compare(updateManifest)
210
211 return (manifest, updateManifest, modifiedAndNew, removed)
212
213#------------------------------------------------------------------------------
214
215def getToremoveFiles(directory):
216 """Add the files to remove from the toremove directory."""
217 toremove = []
218 toremoveDirectory = os.path.join(directory, toremoveName)
219 if os.path.isdir(toremoveDirectory):
220 for entry in os.listdir(toremoveDirectory):
221 toremove.append(os.path.join(toremoveName, entry))
222 return toremove
223
224#------------------------------------------------------------------------------
225
226def createLocalPath(directory, path):
227 """Create a local path from the given manifest path."""
228 localPath = directory
229 for element in path.split("/"):
230 localPath = os.path.join(localPath, element)
231 return localPath
232
233#------------------------------------------------------------------------------
234
235def getToremoveDir(toremoveDir, directory):
236 """Get the path of the directory that will contain the files that are to be
237 removed."""
238 if toremoveDir is None:
239 toremoveDir = os.path.join(directory, toremoveName)
240 try:
241 os.mkdir(toremoveDir)
242 except:
243 pass
244
245 return toremoveDir
246
247#------------------------------------------------------------------------------
248
249def removeFile(toremoveDir, directory, path):
250 """Remove the file at the given path or store it in a temporary directory.
251
252 If the removal of the file fails, it will be stored in a temporary
253 directory. This is useful for files thay may be open and cannot be removed
254 right away."""
255 try:
256 os.remove(path)
257 except:
258 try:
259 sum = hashlib.md5()
260 sum.update(path)
261 toremoveDir = getToremoveDir(toremoveDir, directory)
262 os.rename(path, os.path.join(toremoveDir, sum.hexdigest()))
263 except Exception, e:
264 print "Cannot remove file " + path + ": " + str(e)
265
266#------------------------------------------------------------------------------
267
268def updateFiles(directory, updateURL, listener,
269 manifest, modifiedAndNew, removed, localRemoved):
270 """Update the files according to the given information."""
271 totalSize = 0
272 for (path, size, sum) in modifiedAndNew:
273 totalSize += size
274
275 listener.setTotalSize(len(modifiedAndNew), totalSize,
276 len(removed), len(localRemoved))
277
278 downloaded = 0
279 fin = None
280 toremoveDir = None
281
282 try:
283 updateURL += "/" + os.name
284
285 for (path, size, sum) in modifiedAndNew:
286 targetFile = createLocalPath(directory, path)
287 targetFile += "."
288 targetFile += sum
289
290 targetDirectory = os.path.dirname(targetFile)
291 if not os.path.isdir(targetDirectory):
292 os.makedirs(targetDirectory)
293
294 with open(targetFile, "wb") as fout:
295 fin = urllib2.urlopen(updateURL + "/" + path)
296 while True:
297 data = fin.read(4096)
298 if not data:
299 break
300 fout.write(data)
301 downloaded += len(data)
302 listener.setDownloaded(downloaded)
303 fin.close()
304 fin = None
305
306 listener.startRenaming()
307 count = 0
308 for (path, size, sum) in modifiedAndNew:
309 targetFile = createLocalPath(directory, path)
310
311 downloadedFile = targetFile + "." + sum
312 if os.name=="nt" and os.path.isfile(targetFile):
313 removeFile(toremoveDir, directory, targetFile)
314 os.rename(downloadedFile, targetFile)
315 count += 1
316 listener.renamed(path, count)
317
318 listener.startRemoving()
319 count = 0
320 removed += localRemoved
321 removed.sort(reverse = True)
322 for path in removed:
323 removePath = createLocalPath(directory, path)
324 removeFile(toremoveDir, directory, removePath)
325
326 removeDirectory = os.path.dirname(removePath)
327 try:
328 os.removedirs(removeDirectory)
329 except:
330 pass
331
332 count += 1
333 listener.removed(path, count)
334
335 listener.writingManifest()
336
337 manifestPath = os.path.join(directory, manifestName)
338 with open(manifestPath, "wt") as f:
339 manifest.writeInto(f)
340
341 listener.done()
342 except Exception, e:
343 print >> sys.stderr, "Error:", str(e)
344 listener.failed(str(e))
345
346#------------------------------------------------------------------------------
347
348def isDirectoryWritable(directory):
349 """Determine if the given directory can be written."""
350 checkFile = os.path.join(directory, "writable.chk")
351 try:
352 f = open(checkFile, "wt")
353 f.close()
354 return True
355 except Exception, e:
356 return False
357 finally:
358 try:
359 os.remove(checkFile)
360 except:
361 pass
362
363#------------------------------------------------------------------------------
364
365def processMLXUpdate(buffer, listener):
366 """Process the given buffer supposedly containing a list of commands."""
367 endsWithLine = buffer[-1]=="\n"
368 lines = buffer.splitlines()
369
370 if endsWithLine:
371 buffer = ""
372 else:
373 buffer = lines[-1]
374 lines = lines[:-1]
375
376 for line in lines:
377 words = line.split("\t")
378 try:
379 command = words[0]
380 if command=="downloadingManifest":
381 listener.downloadingManifest()
382 elif command=="downloadedManifest":
383 listener.downloadedManifest()
384 elif command=="setTotalSize":
385 listener.setTotalSize(int(words[1]), long(words[2]),
386 int(words[3]), int(words[4]))
387 elif command=="setDownloaded":
388 listener.setDownloaded(long(words[1]))
389 elif command=="startRenaming":
390 listener.startRenaming()
391 elif command=="renamed":
392 listener.renamed(words[1], int(words[2]))
393 elif command=="startRemoving":
394 listener.startRemoving()
395 elif command=="removed":
396 listener.removed(words[1], int(words[2]))
397 elif command=="writingManifest":
398 listener.writingManifest()
399 elif command=="done":
400 listener.done()
401 elif command=="failed":
402 listener.failed(words[1])
403 except Exception, e:
404 print >> sys.stderr, "Failed to parse line '%s': %s" % \
405 (line, str(e))
406
407 return buffer
408
409#------------------------------------------------------------------------------
410
411def sudoUpdate(directory, updateURL, listener, manifest):
412 """Perform the update via the mlxupdate program."""
413 manifestFD = None
414 manifestFile = None
415 serverSocket = None
416 mlxUpdateSocket = None
417 try:
418 (manifestFD, manifestFile) = tempfile.mkstemp()
419 f = os.fdopen(manifestFD, "wt")
420 try:
421 manifest.writeInto(f)
422 finally:
423 f.close()
424 manifestFD = None
425
426 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
427 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
428 serverSocket.bind(("127.0.0.1", 0))
429 (_host, port) = serverSocket.getsockname()
430 serverSocket.listen(1)
431
432
433 if os.name=="nt":
434 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
435 str(port) + " " + manifestFile, "", 1)
436 else:
437 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
438 str(port), manifestFile],
439 shell = os.name=="nt")
440
441 (mlxUpdateSocket, _) = serverSocket.accept()
442 serverSocket.close()
443 serverSocket = None
444
445 buffer = ""
446 while True:
447 data = mlxUpdateSocket.recv(4096)
448 if not data:
449 break;
450
451 buffer += data
452 buffer = processMLXUpdate(buffer, listener)
453
454 mlxUpdateSocket.close()
455 mlxUpdateSocket = None
456
457 if os.name!="nt":
458 process.wait()
459
460 except Exception, e:
461 print >> sys.stderr, "Failed updating:", str(e)
462 listener.failed(str(e))
463 finally:
464 if serverSocket is not None:
465 try:
466 serverSocket.close()
467 except:
468 pass
469 if mlxUpdateSocket is not None:
470 try:
471 mlxUpdateSocket.close()
472 except:
473 pass
474 if manifestFD is not None:
475 try:
476 os.close(manifestFD)
477 except:
478 pass
479 if manifestFile is not None:
480 try:
481 os.remove(manifestFile)
482 except:
483 pass
484
485
486#------------------------------------------------------------------------------
487
488def update(directory, updateURL, listener, fromGUI = False):
489 """Perform the update."""
490 result = prepareUpdate(directory, updateURL, listener)
491 if result is None:
492 return
493
494 (manifest, updateManifest, modifiedAndNew, removed) = result
495 localRemoved = getToremoveFiles(directory)
496
497 if not modifiedAndNew and not removed and not localRemoved:
498 listener.done()
499 return
500
501 if fromGUI and not isDirectoryWritable(directory):
502 if listener.needSudo():
503 sudoUpdate(directory, updateURL, listener, updateManifest)
504 else:
505 updateFiles(directory, updateURL, listener, updateManifest,
506 modifiedAndNew, removed, localRemoved)
507
508#------------------------------------------------------------------------------
509
510def updateProcess():
511 """This is called in the child process, when we need a child process."""
512 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
513 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
514
515 config = Config()
516 config.load()
517
518 directory = os.path.dirname(sys.argv[0])
519
520 manifest = readLocalManifest(directory)
521
522 updateManifest = Manifest()
523 with open(sys.argv[2], "rt") as f:
524 updateManifest.readFrom(f)
525
526 (modifiedAndNew, removed) = manifest.compare(updateManifest)
527 localRemoved = getToremoveFiles(directory)
528
529 updateFiles(directory, config.updateURL,
530 ClientListener(clientSocket),
531 updateManifest, modifiedAndNew, removed, localRemoved)
532
533#------------------------------------------------------------------------------
534
535def buildManifest(directory):
536 """Build a manifest from the contents of the given directory, into the
537 given directory."""
538
539 manifestPath = os.path.join(directory, manifestName)
540 try:
541 os.remove(manifestPath)
542 except:
543 pass
544
545 manifest = Manifest()
546 manifest.addFiles(directory, [])
547 with open(manifestPath, "wt") as f:
548 manifest.writeInto(f)
549
550#------------------------------------------------------------------------------
551
552# if __name__ == "__main__":
553# manifest1 = Manifest()
554# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
555# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
556# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
557
558# with open("manifest1", "wt") as f:
559# manifest1.writeInto(f)
560
561# manifest2 = Manifest()
562# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
563# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
564# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
565
566# with open("manifest2", "wt") as f:
567# manifest2.writeInto(f)
568
569# manifest1 = Manifest()
570# with open("manifest1", "rt") as f:
571# manifest1.readFrom(f)
572
573# manifest2 = Manifest()
574# with open("manifest2", "rt") as f:
575# manifest2.readFrom(f)
576
577# (modifiedAndNew, removed) = manifest1.compare(manifest2)
578
579# for (path, size, sum) in modifiedAndNew:
580# print "modified or new:", path, size, sum
581
582# for path in removed:
583# print "removed:", path
584
585#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.