source: src/mlx/update.py@ 948:483488585dd8

python3
Last change on this file since 948:483488585dd8 was 948:483488585dd8, checked in by István Váradi <ivaradi@…>, 6 years ago

Encoding handling is fixed during update (re #347).
File size: 21.8 KB
Line 
1
2from .config import Config
3from .util import utf2unicode
4
5import os
6import sys
7import urllib.request, urllib.error, urllib.parse
8import tempfile
9import socket
10import subprocess
11import hashlib
12import traceback
13import io
14
15if os.name=="nt":
16 import win32api
17
18#------------------------------------------------------------------------------
19
20## @package mlx.update
21#
22# Automatic update handling.
23#
24# The program can update itself automatically. For this purpose it maintains a
25# manifest of the files installed containing the relative paths, sizes and
26# checksums of each files. When the program starts up, this manifest file is
27# downloaded from the update server and is compared to the local one. Then the
28# updated and new files are downloaded with names that are created by appending
29# the checksum to the actual name, so as not to overwrite any existing files at
30# this stage. If all files are downloaded, the downloaded files are renamed to
31# their real names. On Windows, the old file is removed first to avoid trouble
32# with 'busy' files. If removing a file fails, the file will be moved to a
33# temporary directory, that will be removed when the program starts the next
34# time.
35
36#------------------------------------------------------------------------------
37
38manifestName = "MLXMANIFEST"
39toremoveName = "toremove"
40
41#------------------------------------------------------------------------------
42
43class Manifest(object):
44 """The manifest of the files.
45
46 The manifest file consists of one line for each file. Each line contains 3
47 items separated by tabs:
48 - the path of the file relative to the distribution's root
49 - the size of the file
50 - the MD5 sum of the file."""
51 def __init__(self):
52 """Construct the manifest."""
53 self._files = {}
54
55 @property
56 def files(self):
57 """Get an iterator over the files.
58
59 Each file is returned as a 3-tuple with items as in the file."""
60 for (path, (size, sum)) in self._files.items():
61 yield (path, size, sum)
62
63 def copy(self):
64 """Create a copy of the manifest."""
65 manifest = Manifest()
66 manifest._files = self._files.copy()
67 return manifest
68
69 def addFile(self, path, size, sum):
70 """Add a file to the manifest."""
71 self._files[path] = (size, sum)
72
73 def addFiles(self, baseDirectory, subdirectory):
74 """Add the files in the given directory and subdirectories of it to the
75 manifest."""
76 directory = baseDirectory
77 for d in subdirectory: directory = os.path.join(directory, d)
78
79 for entry in os.listdir(directory):
80 fullPath = os.path.join(directory, entry)
81 if os.path.isfile(fullPath):
82 size = os.stat(fullPath).st_size
83 sum = hashlib.md5()
84 with open(fullPath, "rb") as f:
85 while True:
86 data = f.read(4096)
87 if data: sum.update(data)
88 else: break
89 self.addFile("/".join(subdirectory + [entry]), size,
90 sum.hexdigest())
91 elif os.path.isdir(fullPath):
92 self.addFiles(baseDirectory, subdirectory + [entry])
93
94 def readFrom(self, file):
95 """Read a manifest from the given file object."""
96 for line in iter(file.readline, ""):
97 (path, size, sum) = line.strip().split("\t")
98 self._files[path] = (int(size), sum)
99
100 def writeInto(self, file):
101 """Write the manifest into the file at the given path."""
102 for (path, (size, sum)) in self._files.items():
103 file.write("%s\t%d\t%s\n" % (path, size, sum))
104
105 def compare(self, other):
106 """Compare this manifest with the other one.
107
108 This returns a tuple of two lists:
109 - the files that are either different or are present in other, but not
110 here. Each file is returned as a 3-tuple as with other functions, the
111 size and sum are those of the new file.
112 - the paths of the files that are present here, but not in other."""
113 modifiedAndNew = []
114 for (path, otherSize, otherSum) in other.files:
115 if path in self._files:
116 (size, sum) = self._files[path]
117 if size!=otherSize or sum!=otherSum:
118 modifiedAndNew.append((path, otherSize, otherSum))
119 else:
120 modifiedAndNew.append((path, otherSize, otherSum))
121
122 removed = [path for path in self._files if path not in other._files]
123
124 return (modifiedAndNew, removed)
125
126 def __contains__(self, path):
127 """Determine if the given path is in the manifest."""
128 return path in self._files
129
130 def __getitem__(self, path):
131 """Get data of the file with the given path."""
132 return self._files[path] if path in self._files else None
133
134#------------------------------------------------------------------------------
135
136class ClientListener(object):
137 """A listener that sends any requests via a socket."""
138 def __init__(self, sock):
139 """Construct the listener."""
140 self._sock = sock
141
142 def downloadingManifest(self):
143 """Called when the downloading of the manifest has started."""
144 self._send(["downloadingManifest"])
145
146 def downloadedManifest(self):
147 """Called when the manifest has been downloaded."""
148 self._send(["downloadedManifest"])
149
150 def needSudo(self):
151 """Called when an admin-level program must be called to complete the
152 update.
153
154 This is not valid from a client, as that client is supposed to be the
155 admin-level program :)"""
156 assert False
157
158 def setTotalSize(self, numToModifyAndNew, totalSize, numToRemove,
159 numToRemoveLocal):
160 """Called when starting the downloading of the files."""
161 self._send(["setTotalSize", str(numToModifyAndNew), str(totalSize),
162 str(numToRemove), str(numToRemoveLocal)])
163
164 def setDownloaded(self, downloaded):
165 """Called periodically after downloading a certain amount of data."""
166 self._send(["setDownloaded", str(downloaded)])
167
168 def startRenaming(self):
169 """Called when the renaming of the downloaded files is started."""
170 self._send(["startRenaming"])
171
172 def renamed(self, path, count):
173 """Called when a file has been renamed."""
174 self._send(["renamed", path, str(count)])
175
176 def startRemoving(self):
177 """Called when the removing of files has started."""
178 self._send(["startRemoving"])
179
180 def removed(self, path, count):
181 """Called when a file has been removed."""
182 self._send(["removed", path, str(count)])
183
184 def writingManifest(self):
185 """Called when we have started writing the manifest."""
186 self._send(["writingManifest"])
187
188 def done(self):
189 """Called when the update has completed."""
190 self._send(["done"])
191
192 def failed(self, what):
193 """Called when something has failed."""
194 self._send(["failed", what])
195
196 def _send(self, words):
197 """Send the given words via the socket."""
198 self._sock.send(bytes("\t".join(words) + "\n", "utf-8"))
199
200#------------------------------------------------------------------------------
201
202def readLocalManifest(directory):
203 """Read the local manifest from the given directory."""
204 manifestPath = os.path.join(directory, manifestName)
205
206 manifest = Manifest()
207 try:
208 with open(manifestPath, "rt") as f:
209 manifest.readFrom(f)
210 except Exception as e:
211 print("Error reading the manifest, ignoring:", utf2unicode(str(e)))
212 manifest = Manifest()
213
214 return manifest
215
216#------------------------------------------------------------------------------
217
218def prepareUpdate(directory, updateURL, listener):
219 """Prepare the update by downloading the manifest and comparing it with the
220 local one."""
221 manifest = readLocalManifest(directory)
222
223 updateURL += "/" + os.name
224
225 listener.downloadingManifest()
226 f = None
227 try:
228 updateManifest = Manifest()
229 reply = urllib.request.urlopen(updateURL + "/" + manifestName)
230 charset = reply.headers.get_content_charset()
231 content = reply.read().decode("utf-8" if charset is None else charset)
232 updateManifest.readFrom(io.StringIO(content))
233
234 except Exception as e:
235 error = utf2unicode(str(e))
236 print("Error downloading manifest:", error, file=sys.stderr)
237 listener.failed(error)
238 return None
239 finally:
240 if f is not None: f.close()
241
242 listener.downloadedManifest()
243
244 (modifiedAndNew, removed) = manifest.compare(updateManifest)
245
246 return (manifest, updateManifest, modifiedAndNew, removed)
247
248#------------------------------------------------------------------------------
249
250def getToremoveFiles(directory):
251 """Add the files to remove from the toremove directory."""
252 toremove = []
253 toremoveDirectory = os.path.join(directory, toremoveName)
254 if os.path.isdir(toremoveDirectory):
255 for entry in os.listdir(toremoveDirectory):
256 toremove.append(os.path.join(toremoveName, entry))
257 return toremove
258
259#------------------------------------------------------------------------------
260
261def createLocalPath(directory, path):
262 """Create a local path from the given manifest path."""
263 localPath = directory
264 for element in path.split("/"):
265 localPath = os.path.join(localPath, element)
266 return localPath
267
268#------------------------------------------------------------------------------
269
270def getToremoveDir(toremoveDir, directory):
271 """Get the path of the directory that will contain the files that are to be
272 removed."""
273 if toremoveDir is None:
274 toremoveDir = os.path.join(directory, toremoveName)
275 try:
276 os.mkdir(toremoveDir)
277 except:
278 pass
279
280 return toremoveDir
281
282#------------------------------------------------------------------------------
283
284def removeFile(toremoveDir, directory, path):
285 """Remove the file at the given path or store it in a temporary directory.
286
287 If the removal of the file fails, it will be stored in a temporary
288 directory. This is useful for files thay may be open and cannot be removed
289 right away."""
290 try:
291 os.remove(path)
292 except:
293 try:
294 sum = hashlib.md5()
295 sum.update(bytes(path, "utf-8"))
296 toremoveDir = getToremoveDir(toremoveDir, directory)
297 targetPath = os.path.join(toremoveDir, sum.hexdigest())
298 try:
299 os.remove(targetPath)
300 except:
301 pass
302 os.rename(path, targetPath)
303 except Exception as e:
304 print("Cannot remove file " + path + ": " + utf2unicode(str(e)))
305
306#------------------------------------------------------------------------------
307
308def removeFiles(directory, listener, removed, count):
309 """Remove the given files."""
310 toremoveDir = None
311
312 listener.startRemoving()
313
314 removed.sort(reverse = True)
315 for path in removed:
316 removePath = createLocalPath(directory, path)
317 removeFile(toremoveDir, directory, removePath)
318
319 removeDirectory = os.path.dirname(removePath)
320 try:
321 os.removedirs(removeDirectory)
322 except:
323 pass
324
325 count += 1
326 listener.removed(path, count)
327
328 return count
329
330#------------------------------------------------------------------------------
331
332def updateFiles(directory, updateURL, listener,
333 manifest, modifiedAndNew, removed, localRemoved):
334 """Update the files according to the given information."""
335 totalSize = 0
336 for (path, size, sum) in modifiedAndNew:
337 totalSize += size
338
339 listener.setTotalSize(len(modifiedAndNew), totalSize,
340 len(removed), len(localRemoved))
341
342 downloaded = 0
343 fin = None
344 toremoveDir = None
345
346 try:
347 updateURL += "/" + os.name
348
349 removeCount = 0
350 if localRemoved:
351 removeCount = removeFiles(directory, listener,
352 localRemoved, removeCount)
353
354 for (path, size, sum) in modifiedAndNew:
355 targetFile = createLocalPath(directory, path)
356 targetFile += "."
357 targetFile += sum
358
359 targetDirectory = os.path.dirname(targetFile)
360 if not os.path.isdir(targetDirectory):
361 os.makedirs(targetDirectory)
362
363 with open(targetFile, "wb") as fout:
364 fin = urllib.request.urlopen(updateURL + "/" + path)
365 while True:
366 data = fin.read(4096)
367 if not data:
368 break
369 fout.write(data)
370 downloaded += len(data)
371 listener.setDownloaded(downloaded)
372 fin.close()
373 fin = None
374
375 listener.startRenaming()
376 count = 0
377 for (path, size, sum) in modifiedAndNew:
378 targetFile = createLocalPath(directory, path)
379
380 downloadedFile = targetFile + "." + sum
381 if os.name=="nt" and os.path.isfile(targetFile):
382 removeFile(toremoveDir, directory, targetFile)
383 os.rename(downloadedFile, targetFile)
384 count += 1
385 listener.renamed(path, count)
386
387 removeFiles(directory, listener, removed, removeCount)
388
389 listener.writingManifest()
390
391 manifestPath = os.path.join(directory, manifestName)
392 with open(manifestPath, "wt") as f:
393 manifest.writeInto(f)
394
395 listener.done()
396 except Exception as e:
397 exc = traceback.format_exc()
398 print(utf2unicode(exc), file=sys.stderr)
399
400 error = utf2unicode(str(e))
401 print("Error:", error, file=sys.stderr)
402
403 listener.failed(error)
404
405#------------------------------------------------------------------------------
406
407def isDirectoryWritable(directory):
408 """Determine if the given directory can be written."""
409 checkFile = os.path.join(directory, "writable.chk")
410 try:
411 f = open(checkFile, "wt")
412 f.close()
413 return True
414 except Exception as e:
415 return False
416 finally:
417 try:
418 os.remove(checkFile)
419 except:
420 pass
421
422#------------------------------------------------------------------------------
423
424def processMLXUpdate(buffer, listener):
425 """Process the given buffer supposedly containing a list of commands."""
426 endsWithLine = buffer[-1]=="\n"
427 lines = buffer.splitlines()
428
429 if endsWithLine:
430 buffer = ""
431 else:
432 buffer = lines[-1]
433 lines = lines[:-1]
434
435 for line in lines:
436 words = line.split("\t")
437 try:
438 command = words[0]
439 if command=="downloadingManifest":
440 listener.downloadingManifest()
441 elif command=="downloadedManifest":
442 listener.downloadedManifest()
443 elif command=="setTotalSize":
444 listener.setTotalSize(int(words[1]), int(words[2]),
445 int(words[3]), int(words[4]))
446 elif command=="setDownloaded":
447 listener.setDownloaded(int(words[1]))
448 elif command=="startRenaming":
449 listener.startRenaming()
450 elif command=="renamed":
451 listener.renamed(words[1], int(words[2]))
452 elif command=="startRemoving":
453 listener.startRemoving()
454 elif command=="removed":
455 listener.removed(words[1], int(words[2]))
456 elif command=="writingManifest":
457 listener.writingManifest()
458 elif command=="done":
459 listener.done()
460 elif command=="failed":
461 listener.failed(words[1])
462 except Exception as e:
463 print("Failed to parse line '%s': %s" % \
464 (line, utf2unicode(str(e))), file=sys.stderr)
465
466 return buffer
467
468#------------------------------------------------------------------------------
469
470def sudoUpdate(directory, updateURL, listener, manifest):
471 """Perform the update via the mlxupdate program."""
472 manifestFD = None
473 manifestFile = None
474 serverSocket = None
475 mlxUpdateSocket = None
476 try:
477 (manifestFD, manifestFile) = tempfile.mkstemp()
478 f = os.fdopen(manifestFD, "wt")
479 try:
480 manifest.writeInto(f)
481 finally:
482 f.close()
483 manifestFD = None
484
485 serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
486 serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
487 serverSocket.bind(("127.0.0.1", 0))
488 (_host, port) = serverSocket.getsockname()
489 serverSocket.listen(1)
490
491
492 if os.name=="nt":
493 win32api.ShellExecute(0, "open", os.path.join(directory, "mlxupdate"),
494 str(port) + " " + manifestFile + " " + updateURL, "", 1)
495 else:
496 process = subprocess.Popen([os.path.join(directory, "mlxupdate"),
497 str(port), manifestFile, updateURL],
498 shell = os.name=="nt")
499
500 (mlxUpdateSocket, _) = serverSocket.accept()
501 serverSocket.close()
502 serverSocket = None
503
504 buffer = ""
505 while True:
506 data = mlxUpdateSocket.recv(4096)
507 if not data:
508 break;
509
510 buffer += str(data, "utf-8")
511 buffer = processMLXUpdate(buffer, listener)
512
513 mlxUpdateSocket.close()
514 mlxUpdateSocket = None
515
516 if os.name!="nt":
517 process.wait()
518
519 except Exception as e:
520 error = utf2unicode(str(e))
521 print("Failed updating:", error, file=sys.stderr)
522 listener.failed(error)
523 finally:
524 if serverSocket is not None:
525 try:
526 serverSocket.close()
527 except:
528 pass
529 if mlxUpdateSocket is not None:
530 try:
531 mlxUpdateSocket.close()
532 except:
533 pass
534 if manifestFD is not None:
535 try:
536 os.close(manifestFD)
537 except:
538 pass
539 if manifestFile is not None:
540 try:
541 os.remove(manifestFile)
542 except:
543 pass
544
545
546#------------------------------------------------------------------------------
547
548def update(directory, updateURL, listener, fromGUI = False):
549 """Perform the update."""
550 try:
551 result = prepareUpdate(directory, updateURL, listener)
552 if result is None:
553 return
554
555 (manifest, updateManifest, modifiedAndNew, removed) = result
556 localRemoved = getToremoveFiles(directory)
557
558 if not modifiedAndNew and not removed and not localRemoved:
559 listener.done()
560 return
561
562 if fromGUI and not isDirectoryWritable(directory):
563 if listener.needSudo():
564 sudoUpdate(directory, updateURL, listener, updateManifest)
565 else:
566 updateFiles(directory, updateURL, listener, updateManifest,
567 modifiedAndNew, removed, localRemoved)
568 except Exception as e:
569 exc = traceback.format_exc()
570 print(utf2unicode(exc), file=sys.stderr)
571
572 error = utf2unicode(str(e))
573 print("Update error:", error, file=sys.stderr)
574
575 listener.failed(error)
576
577#------------------------------------------------------------------------------
578
579def updateProcess():
580 """This is called in the child process, when we need a child process."""
581 try:
582 clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
583 clientSocket.connect(("127.0.0.1", int(sys.argv[1])))
584
585 directory = os.path.dirname(sys.argv[0])
586
587 manifest = readLocalManifest(directory)
588
589 updateManifest = Manifest()
590 with open(sys.argv[2], "rt") as f:
591 updateManifest.readFrom(f)
592
593 (modifiedAndNew, removed) = manifest.compare(updateManifest)
594 localRemoved = getToremoveFiles(directory)
595
596 updateFiles(directory, sys.argv[3],
597 ClientListener(clientSocket),
598 updateManifest, modifiedAndNew, removed, localRemoved)
599 except:
600 exc = traceback.format_exc()
601 print(utf2unicode(exc), file=sys.stderr)
602
603#------------------------------------------------------------------------------
604
605def buildManifest(directory):
606 """Build a manifest from the contents of the given directory, into the
607 given directory."""
608
609 manifestPath = os.path.join(directory, manifestName)
610 try:
611 os.remove(manifestPath)
612 except:
613 pass
614
615 manifest = Manifest()
616 manifest.addFiles(directory, [])
617 with open(manifestPath, "wt") as f:
618 manifest.writeInto(f)
619
620#------------------------------------------------------------------------------
621
622# if __name__ == "__main__":
623# manifest1 = Manifest()
624# manifest1.addFile("file1.exe", 3242, "40398754589435345934")
625# manifest1.addFile("dir/file2.zip", 45645, "347893245873456987")
626# manifest1.addFile("dir/file3.txt", 123, "3432434534534534")
627
628# with open("manifest1", "wt") as f:
629# manifest1.writeInto(f)
630
631# manifest2 = Manifest()
632# manifest2.addFile("file1.exe", 4353, "390734659834756349876")
633# manifest2.addFile("dir/file2.zip", 45645, "347893245873456987")
634# manifest2.addFile("dir/file4.log", 2390, "56546546546546")
635
636# with open("manifest2", "wt") as f:
637# manifest2.writeInto(f)
638
639# manifest1 = Manifest()
640# with open("manifest1", "rt") as f:
641# manifest1.readFrom(f)
642
643# manifest2 = Manifest()
644# with open("manifest2", "rt") as f:
645# manifest2.readFrom(f)
646
647# (modifiedAndNew, removed) = manifest1.compare(manifest2)
648
649# for (path, size, sum) in modifiedAndNew:
650# print "modified or new:", path, size, sum
651
652# for path in removed:
653# print "removed:", path
654
655#------------------------------------------------------------------------------
Note: See TracBrowser for help on using the repository browser.