source: src/mlx/update.py@ 36:f79362793664

Last change on this file since 36:f79362793664 was 36:f79362793664, checked in by István Váradi <ivaradi@…>, 12 years ago

Update seems to work
File size: 16.0 KB
Line 
1# Module handling the download of updates
2
3#------------------------------------------------------------------------------
4
5from config import Config
6
7import os
8import sys
9import urllib2
10import tempfile
11import socket
12import subprocess
13
14if os.name=="nt":
15 import win32api
16
17#------------------------------------------------------------------------------
18
19manifestName = "MLXMANIFEST"
20
21#------------------------------------------------------------------------------
22
23class Manifest(object):
24 """The manifest of the files.
25
26 The manifest file consists of one line for each file. Each line contains 3
27 items separated by tabs:
28 - the path of the file relative to the distribution's root
29 - the size of the file
30 - the MD5 sum of the file."""
31 def __init__(self):
32 """Construct the manifest."""
33 self._files = {}
34
35 @property
36 def files(self):
37 """Get an iterator over the files.
38
39 Each file is returned as a 3-tuple with items as in the file."""
40 for (path, (size, sum)) in self._files.iteritems():
41 yield (path, size, sum)
42
43 def addFile(self, path, size, sum):
44 """Add a file to the manifest."""
45 self._files[path] = ( size, sum)
46
47 def readFrom(self, file):
48 """Read a manifest from the given file object."""
49 for line in iter(file.readline, ""):
50 (path, size, sum) = line.strip().split("\t")
51 self._files[path] = (int(size), sum)
52
53 def writeInto(self, file):
54 """Write the manifest into the file at the given path."""
55 for (path, (size, sum)) in self._files.iteritems():
56 file.write("%s\t%d\t%s\n" % (path, size, sum))
57
58 def compare(self, other):
59 """Compare this manifest with the other one.
60
61 This returns a tuple of two lists:
62 - the files that are either different or are present in other, but not
63 here. Each file is returned as a 3-tuple as with other functions, the
64 size and sum are those of the new file.
65 - the paths of the files that are present here, but not in other."""
66 modifiedAndNew = []
67 for (path, otherSize, otherSum) in other.files:
68 if path in self._files:
69 (size, sum) = self._files[path]
70 if size!=otherSize or sum!=otherSum:
71 modifiedAndNew.append((path, otherSize, otherSum))
72 else:
73 modifiedAndNew.append((path, otherSize, otherSum))
74
75 removed = [path for path in self._files if path not in other._files]
76 removed.sort(reverse = True)
77
78 return (modifiedAndNew, removed)
79
80#------------------------------------------------------------------------------
81
82class ClientListener(object):
83 """A listener that sends any requests via a socket."""
84 def __init__(self, sock):
85 """Construct the listener."""
86 self._sock = sock
87
88 def downloadingManifest(self):
89 """Called when the downloading of the manifest has started."""
90 self._send(["downloadingManifest"])
91
92 def downloadedManifest(self):
93 """Called when the manifest has been downloaded."""
94 self._send(["downloadedManifest"])
95
96 def needSudo(self):
97 """Called when an admin-level program must be called to complete the
98 update.
99
100 This is not valid from a client, as that client is supposed to be the
101 admin-level program :)"""
102 assert False
103
104 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove):
105 """Called when starting the downloading of the files."""
106 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
107 str(numToRemove)])
108
109 def setDownloaded(self, downloaded):
110 """Called periodically after downloading a certain amount of data."""
111 self._send(["setDownloaded", str(downloaded)])
112
113 def startRenaming(self):
114 """Called when the renaming of the downloaded files is started."""
115 self._send(["startRenaming"])
116
117 def renamed(self, path, count):
118 """Called when a file has been renamed."""
119 self._send(["renamed", path, str(count)])
120
121 def startRemoving(self):
122 """Called when the removing of files has started."""
123 self._send(["startRemoving"])
124
125 def removed(self, path, count):
126 """Called when a file has been removed."""
127 self._send(["removed", path, str(count)])
128
129 def writingManifest(self):
130 """Called when we have started writing the manifest."""
131 self._send(["writingManifest"])
132
133 def done(self):
134 """Called when the update has completed."""
135 self._send(["done"])
136
137 def failed(self, what):
138 """Called when something has failed."""
139 self._send(["failed", what])
140
141 def _send(self, words):
142 """Send the given words via the socket."""
143 self._sock.send("\t".join(words) + "\n")
144
145#------------------------------------------------------------------------------
146
147def readLocalManifest(directory):
148 """Read the local manifest from the given directory."""
149 manifestPath = os.path.join(directory, manifestName)
150
151 manifest = Manifest()
152 try:
153 with open(manifestPath, "rt") as f:
154 manifest.readFrom(f)
155 except Exception, e:
156 print "Error reading the manifest, ignoring:", str(e)
157 manifest = Manifest()
158
159 return manifest
160
161#------------------------------------------------------------------------------
162
163def prepareUpdate(directory, updateURL, listener):
164 """Prepare the update by downloading the manifest and comparing it with the
165 local one."""
166 manifest = readLocalManifest(directory)
167
168 updateURL += "/" + os.name
169
170 listener.downloadingManifest()
171 f = None
172 try:
173 updateManifest = Manifest()
174 f= urllib2.urlopen(updateURL + "/" + manifestName)
175 updateManifest.readFrom(f)
176
177 except Exception, e:
178 print >> sys.stderr, "Error downloading manifest:", str(e)
179 listener.failed(str(e))
180 return None
181 finally:
182 if f is not None: f.close()
183
184 listener.downloadedManifest()
185
186 (modifiedAndNew, removed) = manifest.compare(updateManifest)
187
188 return (manifest, updateManifest, modifiedAndNew, removed)
189
190#------------------------------------------------------------------------------
191
192def createLocalPath(directory, path):
193 """Create a local path from the given manifest path."""
194 localPath = directory
195 for element in path.split("/"):
196 localPath = os.path.join(localPath, element)
197 return localPath
198
199#------------------------------------------------------------------------------
200
201def updateFiles(directory, updateURL, listener,
202 manifest, modifiedAndNew, removed):
203 """Update the files according to the given information."""
204 totalSize = 0
205 for (path, size, sum) in modifiedAndNew:
206 totalSize += size
207
208 listener.setTotalSize(len(modifiedAndNew), totalSize, len(removed))
209
210 downloaded = 0
211 fin = None
212 try:
213 updateURL += "/" + os.name
214
215 for (path, size, sum) in modifiedAndNew:
216 targetFile = createLocalPath(directory, path)
217 targetFile += "."
218 targetFile += sum
219
220 targetDirectory = os.path.dirname(targetFile)
221 if not os.path.isdir(targetDirectory):
222 os.makedirs(targetDirectory)
223
224 with open(targetFile, "wb") as fout:
225 fin = urllib2.urlopen(updateURL + "/" + path)
226 while True:
227 data = fin.read(4096)
228 if not data:
229 break
230 fout.write(data)
231 downloaded += len(data)
232 listener.setDownloaded(downloaded)
233 fin.close()
234 fin = None
235
236 listener.startRenaming()
237 count = 0
238 for (path, size, sum) in modifiedAndNew:
239 targetFile = createLocalPath(directory, path)
240
241 downloadedFile = targetFile + "." + sum
242 if os.name=="nt" and os.path.isfile(targetFile):
243 try:
244 os.remove(targetFile)
245 except:
246 try:
247 os.remove(targetFile + ".tmp")
248 except:
249 pass
250 os.rename(targetFile, targetFile + ".tmp")
251 os.rename(downloadedFile, targetFile)
252 count += 1
253 listener.renamed(path, count)
254
255 listener.startRemoving()
256 count = 0
257 for path in removed:
258 os.remove(path)
259 count += 1
260 pathDirectory = os.path.dirname(path)
261 try:
262 os.rmdir(pathDirectory)
263 except:
264 pass
265 listener.removed(path, count)
266
267 listener.writingManifest()
268
269 manifestPath = os.path.join(directory, manifestName)
270 with open(manifestPath, "wt") as f:
271 manifest.writeInto(f)
272
273 listener.done()
274 except Exception, e:
275 print >> sys.stderr, "Error:", str(e)
276 listener.failed(str(e))
277
278#------------------------------------------------------------------------------
279
280def isDirectoryWritable(directory):
281 """Determine if the given directory can be written."""
282 checkFile = os.path.join(directory, "writable.chk")
283 try:
284 f = open(checkFile, "wt")
285 f.close()
286 return True
287 except Exception, e:
288 return False
289 finally:
290 try:
291 os.remove(checkFile)
292 except:
293 pass
294
295#------------------------------------------------------------------------------
296
297def processMLXUpdate(buffer, listener):
298 """Process the given buffer supposedly containing a list of commands."""
299 endsWithLine = buffer[-1]=="\n"
300 lines = buffer.splitlines()
301
302 if endsWithLine:
303 buffer = ""
304 else:
305 buffer = lines[-1]
306 lines = lines[:-1]
307
308 for line in lines:
309 words = line.split("\t")
310 try:
311 command = words[0]
312 if command=="downloadingManifest":
313 listener.downloadingManifest()
314 elif command=="downloadedManifest":
315 listener.downloadedManifest()
316 elif command=="setTotalSize":
317 listener.setTotalSize(int(words[1]), long(words[2]),
318 int(words[3]))
319 elif command=="setDownloaded":
320 listener.setDownloaded(long(words[1]))
321 elif command=="startRenaming":
322 listener.startRenaming()
323 elif command=="renamed":
324 listener.renamed(words[1], int(words[2]))
325 elif command=="startRemoving":
326 listener.startRemoving()
327 elif command=="removed":
328 listener.removed(words[1], int(words[2]))
329 elif command=="writingManifest":
330 listener.writingManifest()
331 elif command=="done":
332 listener.done()
333 elif command=="failed":
334 listener.failed(words[1])
335 except Exception, e:
336 print >> sys.stderr, "Failed to parse line '%s': %s" % \
337 (line, str(e))
338
339 return buffer
340
341#------------------------------------------------------------------------------
342
343def sudoUpdate(directory, updateURL, listener, manifest):
344 """Perform the update via the mlxupdate program."""
345 manifestFD = None
346 manifestFile = None
347 serverSocket = None
348 mlxUpdateSocket = None
349 try:
350 (manifestFD, manifestFile) = tempfile.mkstemp()
351 f = os.fdopen(manifestFD, "wt")
352 try:
353 manifest.writeInto(f)
354 finally:
355 f.close()
356 manifestFD = None
357
358 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
359 serverSocket.bind(("127.0.0.1", 0))
360 (_host, port) = serverSocket.getsockname()
361 serverSocket.listen(1)
362
363 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
364 str(port), manifestFile],
365 shell = os.name=="nt")
366
367 (mlxUpdateSocket, _) = serverSocket.accept()
368 serverSocket.close()
369 serverSocket = None
370
371 buffer = ""
372 while True:
373 data = mlxUpdateSocket.recv(4096)
374 if not data:
375 break;
376
377 buffer += data
378 buffer = processMLXUpdate(buffer, listener)
379
380 mlxUpdateSocket.close()
381 mlxUpdateSocket = None
382
383 process.wait()
384 except Exception, e:
385 print >> sys.stderr, "Failed updating:", str(e)
386 listener.failed(str(e))
387 finally:
388 if serverSocket is not None:
389 try:
390 serverSocket.close()
391 except:
392 pass
393 if mlxUpdateSocket is not None:
394 try:
395 mlxUpdateSocket.close()
396 except:
397 pass
398 if manifestFD is not None:
399 try:
400 os.close(manifestFD)
401 except:
402 pass
403 if manifestFile is not None:
404 try:
405 os.remove(manifestFile)
406 except:
407 pass
408
409
410#------------------------------------------------------------------------------
411
412def update(directory, updateURL, listener, fromGUI = False):
413 """Perform the update."""
414 result = prepareUpdate(directory, updateURL, listener)
415 if result is None:
416 return
417
418 (manifest, updateManifest, modifiedAndNew, removed) = result
419
420 if not modifiedAndNew and not removed:
421 listener.done()
422 return
423
424 if fromGUI and not isDirectoryWritable(directory):
425 if listener.needSudo():
426 sudoUpdate(directory, updateURL, listener,
427 updateManifest)
428 else:
429 updateFiles(directory, updateURL, listener, updateManifest,
430 modifiedAndNew, removed)
431
432#------------------------------------------------------------------------------
433
434def restart():
435 """Restart the program."""
436 programPath = os.path.join(os.path.dirname(sys.argv[0]),
437 "runmlx.exe" if os.name=="nt" else "runmlx.sh")
438 if os.name=="nt":
439 programPath = win32api.GetShortPathName(programPath)
440 os.execl(programPath, programPath)
441
442#------------------------------------------------------------------------------
443
444def updateProcess():
445 """This is called in the child process, when we need a child process."""
446 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
447 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
448
449 config = Config()
450
451 directory = os.path.dirname(sys.argv[0])
452
453 manifest = readLocalManifest(directory)
454
455 updateManifest = Manifest()
456 with open(sys.argv[2], "rt") as f:
457 updateManifest.readFrom(f)
458
459 (modifiedAndNew, removed) = manifest.compare(updateManifest)
460
461 updateFiles(directory, config.updateURL,
462 ClientListener(clientSocket),
463 updateManifest, modifiedAndNew, removed)
464
465#------------------------------------------------------------------------------
466
467if __name__ == "__main__":
468 manifest1 = Manifest()
469 manifest1.addFile("file1.exe", 3242, "40398754589435345934")
470 manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
471 manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
472
473 with open("manifest1", "wt") as f:
474 manifest1.writeInto(f)
475
476 manifest2 = Manifest()
477 manifest2.addFile("file1.exe", 4353, "390734659834756349876")
478 manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
479 manifest2.addFile("dir/file4.log", 2390, "56546546546546")
480
481 with open("manifest2", "wt") as f:
482 manifest2.writeInto(f)
483
484 manifest1 = Manifest()
485 with open("manifest1", "rt") as f:
486 manifest1.readFrom(f)
487
488 manifest2 = Manifest()
489 with open("manifest2", "rt") as f:
490 manifest2.readFrom(f)
491
492 (modifiedAndNew, removed) = manifest1.compare(manifest2)
493
494 for (path, size, sum) in modifiedAndNew:
495 print "modified or new:", path, size, sum
496
497 for path in removed:
498 print "removed:", path
499
500#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.