You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

pyfll 97KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540
  1. #!/usr/bin/python -tt
  2. # -*- coding: utf-8 -*-
  3. __author__ = 'Kel Modderman'
  4. __copyright__ = '(C) 2008 Kel Modderman <kel@otaku42.de>'
  5. __license__ = 'GPLv2 or any later version'
  6. from configobj import ConfigObj
  7. from subprocess import *
  8. import apt_pkg
  9. import apt
  10. import atexit
  11. import datetime
  12. import fileinput
  13. import glob
  14. import logging
  15. import optparse
  16. import os
  17. import sys
  18. import shutil
  19. import signal
  20. import stat
  21. import tempfile
  22. def restore_sigpipe():
  23. """Convenience function so that subprocess may be executed with
  24. SIGPIPE restored to default (http://bugs.python.org/issue1652)."""
  25. signal.signal(signal.SIGPIPE, signal.SIG_DFL)
  26. FLL_LOCALE_DEFAULTS = {
  27. 'be': 'be_BY',
  28. 'cs': 'cs_CZ',
  29. 'da': 'da_DK',
  30. 'en': 'en_US',
  31. 'el': 'el_GR',
  32. 'ga': 'ga_IE',
  33. 'he': 'he_IL',
  34. 'ja': 'ja_JP',
  35. 'ko': 'ko_KR',
  36. 'nb': 'nb_NO',
  37. 'nn': 'nn_NO',
  38. 'pt': 'pt_BR',
  39. 'sl': 'sl_SI',
  40. 'zh': 'zh_CN'
  41. }
  42. class FllLocalesError(Exception):
  43. """
  44. An FllError class for use by FllLocales.
  45. """
  46. pass
  47. class FllLocales(object):
  48. """
  49. A class which provides the ability to determine lists of locale specific
  50. Debian packages using it's detect_locale_packages method.
  51. Arguments:
  52. cache - an apt_pkg cache object
  53. packages - a list or dict of package names which are installed, or are
  54. going to be installed. Locale specific packages are selected
  55. for packages in this data structure.
  56. map - a dict which maps package names with a list of package prefixes
  57. from which the locale string pattern matching can be used
  58. to match locale support packages. The prefered input for map is:
  59. ConfigObj('data/fll-locales-pkg-map').
  60. """
  61. def __init__(self, cache, packages, map):
  62. self.loc_pkgs_set = set()
  63. for pkg in cache.packages:
  64. if not pkg.version_list:
  65. continue
  66. if pkg.name not in packages:
  67. continue
  68. for loc_pkg in map.keys():
  69. if pkg.name == loc_pkg:
  70. loc_pkg_prefix_list = map.get(loc_pkg)
  71. for loc_pkg_prefix in loc_pkg_prefix_list:
  72. self.loc_pkgs_set.add(loc_pkg_prefix)
  73. break
  74. self.loc_pkgs_list_dict = dict()
  75. for loc_pkg in self.loc_pkgs_set:
  76. self.loc_pkgs_list_dict[loc_pkg] = list()
  77. for pkg in cache.packages:
  78. if not pkg.version_list:
  79. continue
  80. for loc_pkg in self.loc_pkgs_set:
  81. if pkg.name.startswith(loc_pkg + '-'):
  82. self.loc_pkgs_list_dict[loc_pkg].append(pkg.name)
  83. def __compute_locale_loc_suf_list(self, locale):
  84. """
  85. Compute a list of locale package name suffixes. The sequence of
  86. suffixes are in preferential order, the lowest index being most
  87. preferential.
  88. This is a very private method, used by detect_locale_packages.
  89. Arguments:
  90. locale - a locale string (eg. en_AU, pt_PT etc.)
  91. """
  92. loc_suf_list = list()
  93. try:
  94. ll, cc = locale.lower().split('_')
  95. except ValueError, e:
  96. raise FllLocalesError(e)
  97. loc_suf_list.append(ll + '-' + cc)
  98. loc_suf_list.append(ll + cc)
  99. loc_suf_list.append(ll)
  100. default = FLL_LOCALE_DEFAULTS.get(ll)
  101. if default and default != locale:
  102. try:
  103. ll, cc = default.lower().split('_')
  104. except ValueError, e:
  105. raise FllLocalesError(e)
  106. loc_suf_list.append(ll + '-' + cc)
  107. loc_suf_list.append(ll + cc)
  108. else:
  109. loc_suf_list.append(ll + '-' + ll)
  110. loc_suf_list.append(ll + ll)
  111. if ll != 'en':
  112. loc_suf_list.append('i18n')
  113. return loc_suf_list
  114. def detect_locale_packages(self, locale):
  115. """
  116. Process the data structures created at FllLocales instantiation and
  117. return a list of package names which are the likely best candidates
  118. for the locale string given as argument.
  119. Arguments:
  120. locale - a locale string (eg. en_AU, pt_PT etc.)
  121. """
  122. suffixes = self.__compute_locale_loc_suf_list(locale)
  123. loc_pkg_dict = dict()
  124. for pkg in self.loc_pkgs_set:
  125. loc_pkgs_list = self.loc_pkgs_list_dict.get(pkg)
  126. if not loc_pkgs_list:
  127. continue
  128. if pkg not in loc_pkg_dict:
  129. loc_pkg_dict[pkg] = dict()
  130. for loc_pkg in loc_pkgs_list:
  131. for idx, suf in enumerate(suffixes):
  132. if loc_pkg == '-'.join([pkg, suf]):
  133. loc_pkg_dict[pkg][idx] = loc_pkg
  134. packages = list()
  135. for pkg in self.loc_pkgs_set:
  136. pkg_candidates = loc_pkg_dict.get(pkg)
  137. if not pkg_candidates:
  138. continue
  139. best = min(pkg_candidates)
  140. packages.append(pkg_candidates[best])
  141. return packages
  142. class FllError(Exception):
  143. '''A generic error handler that does nothing.'''
  144. pass
  145. class FLLBuilder(object):
  146. env = {'LANGUAGE': 'C', 'LC_ALL': 'C', 'LANG' : 'C', 'HOME': '/root',
  147. 'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'SHELL': '/bin/bash',
  148. 'DEBIAN_FRONTEND': 'noninteractive', 'DEBIAN_PRIORITY': 'critical',
  149. 'DEBCONF_NOWARNINGS': 'yes'}
  150. diverts = ['/usr/sbin/policy-rc.d',
  151. '/sbin/modprobe',
  152. '/sbin/insmod',
  153. '/usr/sbin/update-grub',
  154. '/usr/sbin/update-initramfs']
  155. def __init__(self, options):
  156. '''Accept options dict, setup logging.'''
  157. self.opts = options
  158. self.conf = None
  159. self.temp = None
  160. self.log = logging.getLogger('log')
  161. self.log.setLevel(logging.DEBUG)
  162. self.time = datetime.datetime.utcnow()
  163. self.uuid = self.time.strftime('%Y-%m-%d-%H-%M-%S-00')
  164. self.timestamp = self.time.strftime('%Y%m%d%H%M')
  165. def __filterList(self, list, dup_warn = True):
  166. '''Return a list containing no duplicate items given a list that
  167. may have duplicate items.'''
  168. d = dict()
  169. for l in list:
  170. if l in d and dup_warn:
  171. self.log.debug('duplicate: %s' % l)
  172. else:
  173. d[l] = True
  174. list = d.keys()
  175. list.sort()
  176. return list
  177. def __lines2list(self, lines):
  178. '''Return a list of stripped strings given a group of line
  179. separated strings'''
  180. return [s.split('#',1)[0].strip() for s in lines.splitlines()
  181. if s.split('#',1)[0].strip()]
  182. def __isexecutable(self, file):
  183. '''Return True is file is executable, False otherwise.'''
  184. try:
  185. mode = os.stat(file)[stat.ST_MODE]
  186. except OSError:
  187. return False
  188. if stat.S_ISREG(mode) and mode & stat.S_IXUSR:
  189. return True
  190. else:
  191. return False
  192. def __prepDir(self, dir):
  193. '''Set up working directories.'''
  194. if not os.path.isdir(dir):
  195. try:
  196. os.makedirs(dir)
  197. os.chown(dir, self.opts.u, self.opts.g)
  198. except:
  199. self.log.exception('failed to create dir: %s' % dir)
  200. raise FllError
  201. return os.path.realpath(dir)
  202. def __initLogger(self, lvl):
  203. '''Set up the logger.'''
  204. fmt = logging.Formatter('%(asctime)s %(levelname)-5s - %(message)s')
  205. out = logging.StreamHandler()
  206. out.setFormatter(fmt)
  207. out.setLevel(lvl)
  208. self.log.addHandler(out)
  209. def __initLogFile(self, file):
  210. '''Set up a log file.'''
  211. file = os.path.realpath(file)
  212. dir = os.path.dirname(file)
  213. self.__prepDir(dir)
  214. try:
  215. fmt = logging.Formatter('%(asctime)s %(levelname)-5s ' +
  216. '%(message)s')
  217. logfile = logging.FileHandler(filename = file, mode = 'w')
  218. logfile.setFormatter(fmt)
  219. logfile.setLevel(logging.DEBUG)
  220. self.log.addHandler(logfile)
  221. os.chown(file, self.opts.u, self.opts.g)
  222. except:
  223. self.log.exception('failed to setup logfile')
  224. raise FllError
  225. def checkOpts(self):
  226. '''Check and provide default class options.'''
  227. if self.opts.d:
  228. self.__initLogger(logging.DEBUG)
  229. else:
  230. self.__initLogger(logging.INFO)
  231. if self.opts.l:
  232. self.__initLogFile(self.opts.l)
  233. if self.opts.c:
  234. if os.path.isfile(self.opts.c):
  235. self.opts.c = os.path.realpath(self.opts.c)
  236. else:
  237. self.log.critical('configuration file does not exist: %s' %
  238. self.opts.c)
  239. raise FllError
  240. else:
  241. self.log.critical('no config file specified on command line')
  242. raise FllError
  243. if self.opts.s:
  244. if not os.path.isdir(self.opts.s):
  245. self.log.critical('share directory not exist: %s' %
  246. self.opts.s)
  247. raise FllError
  248. self.opts.s = os.path.realpath(self.opts.s)
  249. if self.opts.o:
  250. self.opts.o = self.__prepDir(self.opts.o)
  251. if self.opts.b:
  252. self.opts.b = self.__prepDir(self.opts.b)
  253. def _processDefaults(self, d):
  254. '''Form a distro-defaults data structure to be written to
  255. /etc/default/distro of each chroot, and used for release name.'''
  256. for k in ['FLL_DISTRO_NAME', 'FLL_IMAGE_DIR', 'FLL_IMAGE_FILE',
  257. 'FLL_MEDIA_NAME', 'FLL_MOUNTPOINT', 'FLL_LIVE_USER',
  258. 'FLL_LIVE_USER_GROUPS', 'FLL_GFXBOOT_THEME']:
  259. if not d.get(k):
  260. self.log.critical("%s' is required in 'distro' section " % k +
  261. "of build conf")
  262. raise FllError
  263. for k in ['FLL_DISTRO_NAME', 'FLL_IMAGE_DIR', 'FLL_IMAGE_FILE',
  264. 'FLL_LIVE_USER', 'FLL_DISTRO_CODENAME_SAFE',
  265. 'FLL_DISTRO_CODENAME_REV_SAFE']:
  266. if not d.get(k):
  267. continue
  268. if not d[k].isalnum():
  269. self.log.critical("'%s' is not alphanumeric: %s" % (k, d[k]))
  270. raise FllError
  271. elif d[k].find(' ') >= 0:
  272. self.log.critical("'%s' contains whitespace: %s" % (k, d[k]))
  273. raise FllError
  274. version = d.get('FLL_DISTRO_VERSION')
  275. if version and version != 'snapshot':
  276. if not d.get('FLL_DISTRO_CODENAME_SAFE'):
  277. self.log.critical("'FLL_DISTRO_VERSION' is set, but " +
  278. "'FLL_DISTRO_CODENAME_SAFE' is not")
  279. raise FllError
  280. for k in ['FLL_DISTRO_CODENAME', 'FLL_DISTRO_CODENAME_REV']:
  281. safe = k + '_SAFE'
  282. if d.get(safe) and not d.get(k):
  283. d[k] = d[safe]
  284. else:
  285. d['FLL_DISTRO_VERSION'] = 'snapshot'
  286. def _getDistroImageFile(self, arch):
  287. '''Return image file that compressed chroot will be archived to.'''
  288. image_file = self.conf['distro']['FLL_IMAGE_FILE']
  289. if arch == 'i386':
  290. image_file += '.686'
  291. else:
  292. image_file += '.%s' % arch
  293. self.log.debug('image_file: %s' % image_file)
  294. return image_file
  295. def _getDistroStamp(self):
  296. '''Return a string suitable for the distro stamp file.'''
  297. d = self.conf['distro']
  298. stamp = ' '.join([d['FLL_DISTRO_NAME'], d['FLL_DISTRO_VERSION']])
  299. if d.get('FLL_DISTRO_VERSION') == 'snapshot':
  300. stamp += ' - %s' % self.conf['packages']['profile']
  301. else:
  302. if d.get('FLL_DISTRO_CODENAME_REV'):
  303. stamp += ' - %s' % d['FLL_DISTRO_CODENAME']
  304. stamp += ' %s -' % d['FLL_DISTRO_CODENAME_REV']
  305. else:
  306. stamp += ' %s -' % d['FLL_DISTRO_CODENAME']
  307. stamp += ' %s' % self.conf['packages']['profile']
  308. stamp += ' - (%s)' % self.timestamp
  309. self.log.debug('stamp: %s' % stamp)
  310. return stamp
  311. def _getDistroMediaName(self):
  312. '''Return a string suitable for the distro stamp file.'''
  313. d = self.conf['distro']
  314. name = '-'.join([d['FLL_DISTRO_NAME'], d['FLL_DISTRO_VERSION']])
  315. if d.get('FLL_DISTRO_VERSION') == 'snapshot':
  316. name += '-%s' % self.conf['packages']['profile']
  317. else:
  318. if d.get('FLL_DISTRO_CODENAME_REV'):
  319. name += '-%s-%s' % (d['FLL_DISTRO_CODENAME_SAFE'],
  320. d['FLL_DISTRO_CODENAME_REV_SAFE'])
  321. else:
  322. name += '-%s' % d['FLL_DISTRO_CODENAME_SAFE']
  323. name += '-%s' % self.conf['packages']['profile']
  324. name += '-' + '-'.join(self.conf['archs'].keys())
  325. name += '-%s' % self.timestamp
  326. self.log.debug('name: %s' % name)
  327. return name
  328. def _processConf(self):
  329. '''Process configuration options.'''
  330. arch = ''
  331. if self.opts.a:
  332. arch = self.opts.a
  333. elif not self.conf.get('archs'):
  334. arch = Popen(['dpkg', '--print-architecture'], preexec_fn=restore_sigpipe,
  335. stdout=PIPE).communicate()[0].rstrip()
  336. if len(arch):
  337. self.log.debug('arch: %s' % arch)
  338. if arch not in self.conf['archs'].keys():
  339. self.conf['archs'] = {arch: dict()}
  340. else:
  341. for arch2 in self.conf['archs'].keys():
  342. if arch != arch2:
  343. del self.conf['archs'][arch2]
  344. for arch in self.conf['archs'].keys():
  345. if 'linux' not in self.conf['archs'][arch]:
  346. if arch == 'i386':
  347. cpu = '486'
  348. else:
  349. cpu = arch
  350. linux = cpu
  351. self.conf['archs'][arch].setdefault('linux', [ linux ])
  352. else:
  353. linux = self.conf['archs'][arch]['linux']
  354. if isinstance(linux, str):
  355. self.conf['archs'][arch]['linux'] = [ linux ]
  356. elif not isinstance(linux, list):
  357. self.log.critical('invalid linux for arch %s in config'
  358. % arch)
  359. raise FllError
  360. for linux in self.conf['archs'][arch]['linux']:
  361. self.log.debug("linux (%s): %s" %
  362. (arch, self.conf['archs'][arch]['linux']))
  363. if len(self.conf['repos'].keys()) < 1:
  364. self.log.critical('no apt repos were specified in build config')
  365. raise FllError
  366. if 'sourcedistro' not in self.conf or \
  367. 'name' not in self.conf['sourcedistro']:
  368. self.opts.N = 'debian'
  369. self.opts.C = 'sid'
  370. else:
  371. self.opts.N = self.conf['sourcedistro']['name']
  372. if 'codename' not in self.conf['sourcedistro']:
  373. self.log.critical('codename undefined in distro section of build config')
  374. raise FllError
  375. else:
  376. self.opts.C = self.conf['sourcedistro']['codename']
  377. if self.opts.N not in self.conf['repos']:
  378. self.log.critical('%s repo not configured in build config' % self.opts.N)
  379. raise FllError
  380. if self.conf['options'].get('apt_cacher'):
  381. if not self.opts.t:
  382. self.opts.t = self.conf['options']['apt_cacher']
  383. self.log.debug('Using %s as apt-cacher uri.' % self.opts.t)
  384. for repo in self.conf['repos'].keys():
  385. if self.opts.t:
  386. self.conf['repos'][repo]['cached'] = '%s/%s' % (self.opts.t,
  387. self.conf['repos'][repo]['uri'].split('//')[1])
  388. self.log.info("Cached Repo %s: %s" %
  389. (self.conf['repos'][repo]['label'],
  390. self.conf['repos'][repo]['cached']))
  391. if self.conf['options'].get('builder_uri'):
  392. if not self.opts.x:
  393. self.opts.x = self.conf['options']['builder_uri']
  394. self.log.debug('Using %s as builder uri.' % self.opts.x)
  395. for repo in self.conf['repos'].keys():
  396. if self.conf['repos'][repo].get('trivial'):
  397. words = ['label', 'uri']
  398. else:
  399. words = ['label', 'uri', 'suite', 'components']
  400. for word in words:
  401. if word not in self.conf['repos'][repo]:
  402. self.log.critical("no '%s' for apt repo '%s'" %
  403. (word, repo))
  404. raise FllError
  405. if self.opts.p:
  406. self.conf['packages']['profile'] = self.opts.p
  407. elif 'profile' not in self.conf['packages']:
  408. self.conf['packages']['profile'] = 'kde'
  409. self.log.debug('profile: %s' % self.conf['packages']['profile'])
  410. if 'i18n' not in self.conf['packages'] or \
  411. not self.__lines2list(self.conf['packages']['i18n']):
  412. self.conf['packages']['i18n'] = 'en_US'
  413. i18n = self.__lines2list(self.conf['packages']['i18n'])
  414. self.log.debug('i18n: %s' % ' '.join(i18n))
  415. if 'hashkey' not in self.conf['options']:
  416. self.log.info('No key for signing ISO hashes!')
  417. self.opts.k = None
  418. else:
  419. self.opts.k = self.conf['options'].get('hashkey')
  420. self.log.debug('Using key %s for signing.' % self.opts.k)
  421. if not 'options' in self.conf:
  422. self.conf['options'] = dict()
  423. if self.conf['options'].get('build_dir'):
  424. if not self.opts.b:
  425. dir = self.conf['options']['build_dir']
  426. self.opts.b = self.__prepDir(dir)
  427. else:
  428. if not self.opts.b:
  429. self.opts.b = self.__prepDir(os.getcwd())
  430. if self.conf['options'].get('output_dir'):
  431. if not self.opts.o:
  432. dir = self.conf['options']['output_dir']
  433. self.opts.o = self.__prepDir(dir)
  434. else:
  435. if not self.opts.o:
  436. self.opts.o = self.__prepDir(os.getcwd())
  437. if self.conf['options'].get('build_log'):
  438. if not self.opts.l:
  439. self.opts.l = self.conf['options']['build_log']
  440. self.__initLogFile(self.opts.l)
  441. if self.conf['options'].get('http_proxy'):
  442. self.env['http_proxy'] = self.conf['options']['http_proxy']
  443. if self.conf['options'].get('ftp_proxy'):
  444. self.env['ftp_proxy'] = self.conf['options']['ftp_proxy']
  445. self.conf['options'].setdefault('preferences_d', None)
  446. self.conf['options'].setdefault('apt_conf_d', None)
  447. self.conf['options'].setdefault('apt_recommends', 'no')
  448. self.conf['options'].setdefault('media_include', None)
  449. self.conf['options'].setdefault('apt_cacher', None)
  450. self.conf['options'].setdefault('builder_uri',None)
  451. if 'distro' in self.conf:
  452. self._processDefaults(self.conf['distro'])
  453. self.log.debug('distro-defaults:')
  454. for k, v in self.conf['distro'].items():
  455. self.log.debug('%s="%s"' % (k, v))
  456. self.log.debug('testing stamp/name:')
  457. self._getDistroStamp()
  458. self._getDistroMediaName()
  459. else:
  460. self.log.critical('distro section not found in build config')
  461. raise FllError
  462. def parseConf(self):
  463. '''Parse build configuration file and return it in a dict.'''
  464. self.log.info('reading configuration file...')
  465. self.conf = ConfigObj(self.opts.c)
  466. self._processConf()
  467. def _processPkgProfile(self, arch, profile, dir):
  468. '''Return a dict, arch string as key and package, debconf and postinst
  469. lists.'''
  470. pkgs = {'debconf': [], 'packages': [], 'postinst': []}
  471. linux_meta = ['linux-image', 'linux-headers']
  472. for kvers in self.conf['archs'][arch]['linux']:
  473. pkgs['packages'].extend(['-'.join([l, kvers]) for l in linux_meta])
  474. pname = os.path.basename(profile)
  475. self.log.debug('processing package profile for %s: %s' % (arch, pname))
  476. pfile = ConfigObj(profile)
  477. if 'desc' in pfile:
  478. for l in self.__lines2list(pfile['desc']):
  479. self.log.debug(' %s' % l)
  480. if 'debconf' in pfile:
  481. self.log.debug('debconf:')
  482. for d in self.__lines2list(pfile['debconf']):
  483. pkgs['debconf'].append(d)
  484. self.log.debug(' %s', d)
  485. if 'debconf' in self.conf['packages']:
  486. self.log.debug('debconf (config):')
  487. for d in self.__lines2list(self.conf['packages']['debconf']):
  488. pkgs['debconf'].append(d)
  489. self.log.debug(' %s' % d)
  490. if 'packages' in pfile:
  491. self.log.debug('packages:')
  492. for p in self.__lines2list(pfile['packages']):
  493. pkgs['packages'].append(p)
  494. self.log.debug(' %s' % p)
  495. if 'packages' in self.conf['packages']:
  496. self.log.debug('packages (config):')
  497. for p in self.__lines2list(self.conf['packages']['packages']):
  498. pkgs['packages'].append(p)
  499. self.log.debug(' %s' % p)
  500. if arch in pfile:
  501. self.log.debug('packages (%s):' % arch)
  502. for p in self.__lines2list(pfile[arch]):
  503. pkgs['packages'].append(p)
  504. self.log.debug(' %s' % p)
  505. deps = ['essential']
  506. if 'deps' in pfile:
  507. self.log.debug('deps:')
  508. for dep in self.__lines2list(pfile['deps']):
  509. deps.append(dep)
  510. self.log.debug(' %s' % dep)
  511. if 'deps' in self.conf['packages']:
  512. self.log.debug('deps (config):')
  513. for dep in self.__lines2list(self.conf['packages']['deps']):
  514. deps.append(dep)
  515. self.log.debug(' %s' % dep)
  516. if os.path.isfile(profile + '.postinst'):
  517. self.log.debug('registering postinst script')
  518. pkgs['postinst'].append(profile + '.postinst')
  519. self.log.debug('---')
  520. for dep in deps:
  521. depfile = os.path.join(dir, 'packages.d', dep)
  522. if not os.path.isfile(depfile):
  523. self.log.critical('no such dep file: %s' % depfile)
  524. raise FllError
  525. dname = os.path.basename(depfile)
  526. self.log.debug('processing dependency file: %s' % dname)
  527. dfile = ConfigObj(depfile)
  528. if 'desc' in dfile:
  529. for l in self.__lines2list(dfile['desc']):
  530. self.log.debug(' %s' % l)
  531. if 'debconf' in dfile:
  532. self.log.debug('debconf:')
  533. for d in self.__lines2list(dfile['debconf']):
  534. pkgs['debconf'].append(d)
  535. self.log.debug(' %s' % d)
  536. if 'packages' in dfile:
  537. self.log.debug('packages:')
  538. for p in self.__lines2list(dfile['packages']):
  539. pkgs['packages'].append(p)
  540. self.log.debug(' %s' % p)
  541. if arch in dfile:
  542. self.log.debug('packages (%s):' % arch)
  543. for p in self.__lines2list(dfile[arch]):
  544. pkgs['packages'].append(p)
  545. self.log.debug(' %s' % p)
  546. if os.path.isfile(depfile + '.postinst'):
  547. self.log.debug('registering postinst script')
  548. pkgs['postinst'].append(depfile + '.postinst')
  549. self.log.debug('---')
  550. self.log.info('last depfile %s' % depfile)
  551. depfile = os.path.join(dir, 'packages.d', 'finalize.postinst')
  552. pkgs['postinst'].append(depfile)
  553. self.log.debug('package summary for %s:' % arch)
  554. pkgs['packages'].sort()
  555. for p in pkgs['packages']:
  556. self.log.debug(' %s' % p)
  557. self.log.debug('debconf summary for %s:' % arch)
  558. pkgs['debconf'].sort()
  559. for d in pkgs['debconf']:
  560. self.log.debug(' %s' % d)
  561. pkgs['packages'] = self.__filterList(pkgs['packages'])
  562. return pkgs
  563. def parsePkgProfile(self):
  564. '''Parse packages profile file(s).'''
  565. self.log.info('processing package profile (%s)...' %
  566. self.conf['packages']['profile'])
  567. dir = os.path.join(self.opts.s, 'packages')
  568. file = os.path.join(dir, self.conf['packages']['profile'])
  569. if not os.path.isfile(file):
  570. self.log.critical('no such package profile file: %s' % file)
  571. raise FllError
  572. self.pkgs = dict()
  573. for arch in self.conf['archs'].keys():
  574. self.pkgs[arch] = self._processPkgProfile(arch, file, dir)
  575. try:
  576. self.pkgs[arch]['packages'] += \
  577. self.__lines2list(self.conf['packages'][arch])
  578. except KeyError:
  579. pass
  580. def _getDebconfList(self, arch):
  581. '''Return debconf list for arch.'''
  582. return self.pkgs[arch]['debconf']
  583. def _getPackageList(self, arch):
  584. '''Return package list for arch.'''
  585. return self.pkgs[arch]['packages']
  586. def _getPostinstList(self, arch):
  587. '''Return postinst list for arch.'''
  588. return self.pkgs[arch]['postinst']
  589. def _stageMedia(self, point, dir, fnames):
  590. '''Copy content from a directory to live media staging area.'''
  591. orig, dest = point
  592. dirname = dir.partition(orig)[2].lstrip('/')
  593. remove = []
  594. for f in fnames:
  595. if f.startswith('.') or f.endswith('~'):
  596. remove.append(f)
  597. elif os.path.isdir(os.path.join(dir, f)) and \
  598. f == 'boot':
  599. remove.append(f)
  600. elif os.path.isdir(os.path.join(dir, f)):
  601. if not os.path.isdir(os.path.join(dest, dirname, f)):
  602. os.mkdir(os.path.join(dest, dirname, f))
  603. else:
  604. if not os.path.isfile(os.path.join(dest, dirname, f)):
  605. shutil.copy(os.path.join(dir, f),
  606. os.path.join(dest, dirname))
  607. for r in remove:
  608. fnames.remove(r)
  609. def stageBuildArea(self):
  610. '''Prepare temporary directory for chroots and result staging area.'''
  611. self.log.debug('preparing build area...')
  612. self.temp = tempfile.mkdtemp(prefix = 'fll_', dir = self.opts.b)
  613. os.chown(self.temp, self.opts.u, self.opts.g)
  614. atexit.register(self.cleanup)
  615. stage = os.path.join(self.temp, 'staging')
  616. os.mkdir(stage)
  617. os.mkdir(os.path.join(stage, 'boot'))
  618. os.mkdir(os.path.join(stage, self.conf['distro']['FLL_IMAGE_DIR']))
  619. if self.conf['options']['media_include']:
  620. media_include = self.conf['options']['media_include']
  621. if os.path.isdir(media_include):
  622. try:
  623. os.path.walk(media_include, self._stageMedia,
  624. (media_include, stage))
  625. except:
  626. self.log.exception('problem copying media_include ' +
  627. 'contents to staging dir')
  628. raise FllError
  629. def _mount(self, chroot):
  630. '''Mount virtual filesystems in a chroot.'''
  631. virtfs = {'devpts': 'dev/pts', 'proc': 'proc'}
  632. for v in virtfs.items():
  633. cmd = ['mount', '-t', v[0], 'fll-' + v[0],
  634. os.path.join(chroot, v[1])]
  635. retv = call(cmd, preexec_fn=restore_sigpipe)
  636. if retv != 0:
  637. self.log.critical('failed to mount chroot %s' % v[0])
  638. raise FllError
  639. def _umount(self, chrootdir):
  640. '''Umount any mount points in a chroot.'''
  641. umount_list = []
  642. try:
  643. for line in open('/proc/mounts'):
  644. (dev, mnt, fs, options, d, p) = line.split()
  645. if mnt.startswith(chrootdir):
  646. umount_list.append(mnt)
  647. except IOError:
  648. self.log.exception('failed to open /proc/mounts')
  649. raise FllError
  650. umount_list.sort(key=len)
  651. umount_list.reverse()
  652. for mpoint in umount_list:
  653. retv = call(['umount', mpoint], preexec_fn=restore_sigpipe)
  654. if retv != 0:
  655. self.log.critical('umount failed for: %s' % mpoint)
  656. raise FllError
  657. def _nuke(self, dir):
  658. '''Nuke directory tree.'''
  659. if os.path.isdir(dir):
  660. self.log.debug('nuking directory: %s' % dir)
  661. try:
  662. shutil.rmtree(dir)
  663. except:
  664. self.log.exception('unable to remove %s' % dir)
  665. raise FllError
  666. else:
  667. self.log.debug('not nuking directory (does not exist): %s' % dir)
  668. def _nukeChroot(self, arch):
  669. '''Convenience function to nuke chroot given by arch name.'''
  670. if not self.opts.P:
  671. self.log.info('nuking %s chroot...' % arch)
  672. chroot = os.path.join(self.temp, arch)
  673. self._umount(chroot)
  674. self._nuke(chroot)
  675. def cleanup(self):
  676. '''Clean up the build area after taking care that all build chroots
  677. have been taken care of.'''
  678. self.log.info('cleaning up...')
  679. for arch in self.conf['archs'].keys():
  680. dir = os.path.join(self.temp, arch)
  681. if os.path.isdir(dir):
  682. self.log.debug('cleaning up %s chroot...' % arch)
  683. self._umount(dir)
  684. if not self.opts.P:
  685. self._nuke(dir)
  686. if not self.opts.P:
  687. self._nuke(self.temp)
  688. def __execLogged(self, cmd, check_returncode):
  689. '''Execute a command logging all output. Output sent to the console is
  690. buffered until the command has finished execution.'''
  691. self.log.debug(' '.join(cmd))
  692. try:
  693. c = Popen(cmd, stdout = PIPE, stderr = STDOUT, env = self.env,
  694. close_fds = True, preexec_fn=restore_sigpipe)
  695. cout = c.communicate()[0]
  696. retv = c.returncode
  697. except KeyboardInterrupt:
  698. raise FllError
  699. except:
  700. self.log.exception('problem executing command: %s' % ' '.join(cmd))
  701. raise FllError
  702. for line in cout.splitlines():
  703. if self.opts.q:
  704. self.log.debug(line.rstrip())
  705. else:
  706. self.log.info(line.rstrip())
  707. if retv != 0 and check_returncode:
  708. self.log.critical('command failed with return value: %d' %
  709. c.returncode)
  710. raise FllError
  711. def __exec(self, cmd, check_returncode):
  712. '''Execute subprocess without buffering output in a pipe.'''
  713. self.log.debug(' '.join(cmd))
  714. try:
  715. if self.opts.q:
  716. retv = call(cmd, stdout = open(os.devnull, 'w'),
  717. stderr = STDOUT, env = self.env,
  718. close_fds = True, preexec_fn=restore_sigpipe)
  719. else:
  720. retv = call(cmd, env = self.env, close_fds = True,
  721. preexec_fn=restore_sigpipe)
  722. except KeyboardInterrupt:
  723. raise FllError
  724. except:
  725. self.log.exception('problem executing command: %s' % ' '.join(cmd))
  726. raise FllError
  727. if retv != 0 and check_returncode:
  728. self.log.critical('command failed with return value: %d' % retv)
  729. raise FllError
  730. def _execCmd(self, cmd, check_returncode = True):
  731. '''Convenience wrapper for subprocess execution.'''
  732. if self.opts.l:
  733. self.__execLogged(cmd, check_returncode)
  734. else:
  735. self.__exec(cmd, check_returncode)
  736. def _execInChroot(self, arch, args, check_returncode = True):
  737. '''Run command in a chroot.'''
  738. chroot = os.path.join(self.temp, arch)
  739. cmd = ['chroot', chroot]
  740. cmd.extend(args)
  741. self._mount(chroot)
  742. if self.opts.l:
  743. self.__execLogged(cmd, check_returncode)
  744. else:
  745. self.__exec(cmd, check_returncode)
  746. self._umount(chroot)
  747. def _aptGetInstall(self, arch, pkgs, download_only = False):
  748. '''An apt-get install wrapper. Automatic installation of recommended
  749. packages defaults to disabled.'''
  750. aptget = ['apt-get', '--yes']
  751. if download_only:
  752. aptget.append('--download-only')
  753. if self.conf['options']['apt_recommends'] == 'no':
  754. aptget.extend(['-o', 'APT::Install-Recommends=0'])
  755. if self.opts.d:
  756. aptget.extend(['-o', 'APT::Get::Show-Versions=1'])
  757. aptget.append('install')
  758. aptget.extend(pkgs)
  759. self._execInChroot(arch, aptget)
  760. def __cdebBootStrap(self, arch, dir, mirror, codename):
  761. '''Bootstrap a debian system with cdebootstrap.'''
  762. cmd = ['cdebootstrap', '--arch=%s' % arch,
  763. '--include=apt-utils,bzip2,gnupg2,dirmngr,xz-utils,perl',
  764. '--exclude=gnupg1',
  765. '--flavour=minimal', codename, dir, mirror]
  766. if self.opts.d:
  767. cmd.append('--debug')
  768. elif self.opts.v:
  769. cmd.append('--verbose')
  770. self._execCmd(cmd)
  771. cmd = 'dpkg --purge cdebootstrap-helper-rc.d'
  772. self._execInChroot(arch, cmd.split())
  773. def __debBootStrap(self, arch, dir, mirror, codename):
  774. '''Bootstrap a debian system with debootstrap.'''
  775. cmd = ['debootstrap', '--arch=%s' % arch,
  776. '--include=apt-utils,bzip2,gnupg2,dirmngr,xz-utils,perl',
  777. '--variant=minbase', codename, dir, mirror]
  778. if self.opts.d or self.opts.v:
  779. cmd.insert(1, '--verbose')
  780. self._execCmd(cmd)
  781. def _bootStrap(self, arch):
  782. '''Bootstrap a debian system with cdebootstrap.'''
  783. distro = self.opts.N
  784. debian = self.conf['repos'][distro]
  785. codename = self.opts.C
  786. if debian.get('cached'):
  787. mirror = debian['cached']
  788. else:
  789. mirror = debian['uri']
  790. dir = os.path.join(self.temp, arch)
  791. self.log.info('bootstrapping %s %s %s...' % (distro,codename,arch))
  792. bootstrapper = self.conf['options'].get('bootstrapper')
  793. if bootstrapper == 'debootstrap':
  794. self.__debBootStrap(arch, dir, mirror, codename)
  795. else:
  796. self.__cdebBootStrap(arch, dir, mirror, codename)
  797. shutil.copy('/etc/hosts', os.path.join(dir, 'etc'))
  798. shutil.copy('/etc/hostname', os.path.join(dir, 'etc'))
  799. os.mkdir(os.path.join(dir, 'disks'), 0755)
  800. def _writeAptLists(self, arch, cached = False, src_uri = False):
  801. '''Write apt source lists to /etc/apt/sources.list.d/*.'''
  802. chroot = os.path.join(self.temp, arch)
  803. for repo in self.conf['repos'].keys():
  804. r = self.conf['repos'][repo]
  805. file = os.path.join(chroot, 'etc/apt/sources.list.d',
  806. r['label'] + '.list')
  807. if os.path.isfile(os.path.join(chroot, 'etc/apt/sources.list')):
  808. s = None
  809. try:
  810. s = open(os.path.join(chroot, 'etc/apt/sources.list'), 'a')
  811. s.write('# %-74s#\n' % file.partition(chroot)[2])
  812. except IOError:
  813. self.log.exception('failed to open /etc/apt/sources.list')
  814. raise FllError
  815. finally:
  816. if s:
  817. s.close()
  818. self.log.debug("creating %s" % file)
  819. line = []
  820. if cached and r.get('cached'):
  821. line.append(r['cached'])
  822. else:
  823. line.append(r['uri'])
  824. if r.get('trivial'):
  825. line.append('./')
  826. else:
  827. line.append(r['suite'])
  828. line.append(r['components'])
  829. line.append("\n")
  830. l = ' '.join(line)
  831. self.log.debug('%s: %s', repo, l.rstrip())
  832. list = None
  833. try:
  834. list = open(file, 'w')
  835. list.write('deb ' + l)
  836. if not src_uri or self.opts.B:
  837. list.write('#deb-src ' + l)
  838. else:
  839. list.write('deb-src ' + l)
  840. except IOError:
  841. self.log.exception('failed to open %s' % file)
  842. raise FllError
  843. finally:
  844. if list:
  845. list.close()
  846. if os.path.isfile(os.path.join(chroot, 'etc/apt/sources.list')):
  847. s = None
  848. try:
  849. s = open(os.path.join(chroot, 'etc/apt/sources.list'), 'a')
  850. s.write('# ' * 39 + '#\n')
  851. except IOError:
  852. self.log.exception('failed to open %s' % file)
  853. raise FllError
  854. finally:
  855. if s:
  856. s.close()
  857. def _primeApt(self, arch):
  858. '''Prepare apt for work in each build chroot. Fetch all required gpg
  859. keys and initialize apt_pkg config.'''
  860. self.log.info('preparing apt in %s chroot...' % arch)
  861. chroot = os.path.join(self.temp, arch)
  862. # copy helper files
  863. try:
  864. files = glob.glob(os.path.join(self.opts.s, 'sbin', '*'))
  865. for file in files:
  866. shutil.copy(file, os.path.join(chroot, 'usr/sbin/'))
  867. except IOError:
  868. self.log.error('failed to copy: %s' % file)
  869. raise FllError
  870. # copy repo keys to /tmp
  871. try:
  872. files = glob.glob(os.path.join(self.opts.s, 'data/repo-keys/', '*.asc'))
  873. for file in files:
  874. shutil.copy(file, os.path.join(chroot, 'tmp/'))
  875. except IOError:
  876. self.log.error('failed to copy: %s' % file)
  877. raise FllError
  878. preferences_d = self.conf['options']['preferences_d']
  879. if preferences_d:
  880. self.log.info('importing apt preferences directory')
  881. try:
  882. files = glob.glob(os.path.join(self.opts.s, 'data', 'preferences.d','%s' % preferences_d, '*'))
  883. for file in files:
  884. shutil.copy(file, os.path.join(chroot, 'etc/apt/preferences.d'))
  885. except IOError:
  886. self.log.error('apt preferences directory failed to copy: %s' %
  887. preferences_d)
  888. raise FllError
  889. apt_conf_d = self.conf['options']['apt_conf_d']
  890. if apt_conf_d:
  891. self.log.info('importing apt configuration directory')
  892. try:
  893. conf_files = glob.glob(os.path.join(self.opts.s, 'data', 'apt.conf.d','%s' % apt_conf_d, '*'))
  894. for file in conf_files:
  895. shutil.copy(file, os.path.join(chroot, 'etc/apt/apt.conf.d'))
  896. except IOError:
  897. self.log.error('apt configuration directory failed to copy: %s'
  898. % apt_conf_d)
  899. raise FllError
  900. self.log.debug('removing sources.list from %s chroot' % arch)
  901. list = os.path.join(chroot, 'etc/apt/sources.list')
  902. if os.path.isfile(list):
  903. os.unlink(list)
  904. self._writeAptLists(arch, cached = True, src_uri = True)
  905. keyrings = []
  906. for repo in self.conf['repos'].keys():
  907. r = self.conf['repos'][repo]
  908. if 'keyring' in r and r['keyring']:
  909. keyrings.append(r['keyring'])
  910. if len(keyrings) > 0:
  911. self._execInChroot(arch, 'apt-get -o Acquire::Languages=none -o Acquire::Check-Valid-Until=false --allow-insecure-repositories update'.split())
  912. cmd = 'apt-get --allow-unauthenticated --yes install'.split()
  913. cmd.extend(keyrings)
  914. self._execInChroot(arch, cmd)
  915. gpgkeys = []
  916. for repo in self.conf['repos'].keys():
  917. r = self.conf['repos'][repo]
  918. if 'gpgkey' in r:
  919. self.log.debug('importing gpg key for %s' % r['label'])
  920. gpgkeys.append(r['gpgkey'])
  921. if r['gpgkey'].startswith('http'):
  922. cmd = 'apt-key adv --fetch-keys ' + r['gpgkey']
  923. self._execInChroot(arch, cmd.split())
  924. elif r['gpgkey'].endswith('asc'):
  925. keyfile = os.path.join('tmp', r['gpgkey'])
  926. self.log.info('keyfile %s' % keyfile)
  927. if os.path.isfile(os.path.join(chroot, keyfile)):
  928. # we copied them to /tmp before - didn't we? :P
  929. cmd = 'apt-key add ' + keyfile
  930. self.log.info('cmd %s' % cmd)
  931. self._execInChroot(arch, cmd.split(),
  932. check_returncode = False)
  933. else:
  934. cmd = 'gpg-receive-key ' + r['gpgkey']
  935. self._execInChroot(arch, cmd.split(), check_returncode = False)
  936. cmd = 'apt-import-key ' + r['gpgkey']
  937. self._execInChroot(arch, cmd.split(), check_returncode = False)
  938. self._execInChroot(arch, 'apt-get -o Acquire::Languages=none -o Acquire::Check-Valid-Until=false update'.split())
  939. # Upgrade any essential packages from sources not available
  940. # during bootstrap phase.
  941. self._execInChroot(arch, 'apt-get -o Acquire::Check-Valid-Until=false dist-upgrade --yes'.split())
  942. apt_pkg.init_config()
  943. apt_pkg.config.set('RootDir', chroot)
  944. apt_pkg.config.set('APT::Architecture', arch)
  945. ## Reset APT::Default-Release, probably set in Hosts apt-config
  946. ## This is more than hacky, because all other Host-Settings will
  947. ## go in. There should be a clean solution within pyfll
  948. apt_pkg.config.set('APT::Default-Release', '');
  949. ## End fugly hacking
  950. apt_pkg.init_system()
  951. def _dpkgAddDivert(self, arch):
  952. '''Divert some facilities and replace temporaily with /bin/true (or
  953. some other more appropiate facility.'''
  954. chroot = os.path.join(self.temp, arch)
  955. for d in self.diverts:
  956. self.log.debug("diverting %s" % d)
  957. cmd = 'dpkg-divert --add --local --divert ' + d + '.REAL --rename '
  958. cmd += d
  959. self._execInChroot(arch, cmd.split())
  960. if d == '/usr/sbin/policy-rc.d':
  961. self._writeFile(arch, d)
  962. os.chmod(os.path.join(chroot, d.lstrip('/')), 0755)
  963. else:
  964. os.symlink('/bin/true', os.path.join(chroot, d.lstrip('/')))
  965. def _dpkgUnDivert(self, arch):
  966. '''Undivert facilities diverted by self._dpkgAddDivert().'''
  967. chroot = os.path.join(self.temp, arch)
  968. for d in self.diverts:
  969. self.log.debug("undoing diversion: %s" % d)
  970. os.unlink(os.path.join(chroot, d.lstrip('/')))
  971. cmd = 'dpkg-divert --remove --rename ' + d
  972. self._execInChroot(arch, cmd.split())
  973. def _writeFile(self, arch, file):
  974. '''Write a file in a chroot. Templates for common files included
  975. below.'''
  976. chroot = os.path.join(self.temp, arch)
  977. f = None
  978. fn = None
  979. mode = 0644
  980. try:
  981. fn = os.path.join(chroot, file.lstrip('/'))
  982. if os.path.isfile(fn):
  983. mode = None
  984. if not os.path.exists(os.path.dirname(fn)):
  985. try:
  986. os.makedirs(os.path.dirname(fn), 0755)
  987. except OSError as exc:
  988. if exc.errno != errno.EEXIST:
  989. raise
  990. f = open(fn, 'w')
  991. self.log.debug('writing file: %s' % file)
  992. if file == '/etc/default/distro':
  993. d = self.conf['distro'].keys()
  994. d.sort()
  995. for k in d:
  996. if k.startswith('FLL_DISTRO_CODENAME'):
  997. f.write('%s="%s"\n' % (k, self.conf['distro'][k]))
  998. elif k == 'FLL_MOUNTPOINT':
  999. f.write('%s="%s"\n' % (k, self.conf['distro'][k]))
  1000. test = '$([ -d "$%s" ] && echo live' % k
  1001. test += ' || echo installed)'
  1002. f.write('%s="%s"\n' % ('FLL_DISTRO_MODE', test))
  1003. elif k == 'FLL_IMAGE_FILE':
  1004. image_file = self._getDistroImageFile(arch)
  1005. f.write('%s="%s"\n' % (k, image_file))
  1006. f.write('%s="$%s/$%s"\n' % ('FLL_IMAGE_LOCATION',
  1007. 'FLL_IMAGE_DIR', k))
  1008. else:
  1009. f.write('%s="%s"\n' % (k, self.conf['distro'][k]))
  1010. elif file == '/etc/fstab':
  1011. f.write('# /etc/fstab: static file system information\n')
  1012. elif file == '/etc/hostname':
  1013. hostname = self.conf['distro']['FLL_DISTRO_NAME']
  1014. f.write(hostname + '\n')
  1015. elif file == '/etc/hosts':
  1016. hostname = self.conf['distro']['FLL_DISTRO_NAME']
  1017. f.write('127.0.0.1\tlocalhost\n')
  1018. f.write('127.0.1.1\t' + hostname +'.local\t' + hostname + '\n\n')
  1019. f.write('# Below lines are for IPv6 capable hosts\n')
  1020. f.write('::1 localhost ip6-localhost ip6-loopback\n')
  1021. f.write('fe00::0 ip6-localnet\n')
  1022. f.write('ff00::0 ip6-mcastprefix\n')
  1023. f.write('ff02::1 ip6-allnodes\n')
  1024. f.write('ff02::2 ip6-allrouters\n')
  1025. f.write('ff02::3 ip6-allhosts\n')
  1026. elif file == '/etc/network/interfaces':
  1027. f.write('# /etc/network/interfaces - ')
  1028. f.write('configuration file for ifup(8), ifdown(8)\n\n')
  1029. f.write('# The loopback interface\n')
  1030. f.write('auto lo\n')
  1031. f.write('iface lo inet loopback\n')
  1032. elif file == '/etc/resolv.conf':
  1033. f.write('# Generated by pyfll\n')
  1034. f.write('nameserver 8.8.8.8\n')
  1035. elif file == "/etc/sudoers.d/15_%s" % self.conf['distro']['FLL_DISTRO_NAME']:
  1036. liveuser= self.conf['distro']['FLL_LIVE_USER']
  1037. f.write('# WARNING: This allows the unprivileged %s user to start commands as root\n' % liveuser)
  1038. f.write('# WARNING: This is totally insecure and (almost) makes %s a second root account.\n' % liveuser)
  1039. f.write('# WARNING: Never allow external access to the %s user!!!\n' % liveuser )
  1040. f.write('%s ALL=(ALL:ALL) NOPASSWD: ALL\n' % liveuser )
  1041. elif file == '/usr/sbin/policy-rc.d':
  1042. f.write('#!/bin/sh\n')
  1043. f.write('echo "$0 denied action: \`$1 $2\'" >&2\n')
  1044. f.write('exit 101\n')
  1045. elif file == '/tmp/iso_uuid':
  1046. f.write(self.uuid)
  1047. self.conf['distro']['FLL_UUID'] = self.uuid
  1048. elif file == '/tmp/fll-data':
  1049. f.write('DISTRO="%s"\n' % (self.conf['distro']['FLL_DISTRO_NAME']))
  1050. f.write('FLAVOUR="%s"\n' % (self.conf['distro']['FLL_FLAVOUR']))
  1051. f.write('CODENAME="%s"\n' % (self.conf['distro']['FLL_DISTRO_CODENAME_SAFE']))
  1052. except IOError:
  1053. self.log.exception('failed to open file for writing: %s' % file)
  1054. raise FllError
  1055. finally:
  1056. if f:
  1057. f.close()
  1058. if mode:
  1059. os.chmod(fn, mode)
  1060. def _defaultEtc(self, arch):
  1061. '''Initial creation of conffiles required in chroot.'''
  1062. self._writeFile(arch, '/etc/fstab')
  1063. self._writeFile(arch, '/etc/network/interfaces')
  1064. self._writeFile(arch, '/etc/resolv.conf')
  1065. # create or recreate /etc/sudoers.d in chroot
  1066. cmd = 'mkdir -p /etc/sudoers.d'
  1067. self._execInChroot(arch, cmd.split())
  1068. self._writeFile(arch, '/etc/sudoers.d/15_%s' % (self.conf['distro']['FLL_DISTRO_NAME']))
  1069. self._writeFile(arch, '/tmp/iso_uuid')
  1070. self._writeFile(arch, '/tmp/fll-data')
  1071. def _distroDefaultEtc(self, arch):
  1072. '''Write the /etc/default/distro file.'''
  1073. self._writeFile(arch, '/etc/default/distro')
  1074. def _finalEtc(self, arch):
  1075. '''Final editing of conffiles in chroot.'''
  1076. chroot = os.path.join(self.temp, arch)
  1077. distro_version = '%s-version' % \
  1078. self.conf['distro']['FLL_DISTRO_NAME'].lower()
  1079. distro_version = os.path.join(chroot, 'etc', distro_version)
  1080. self.log.debug('stamping distro version: %s' % distro_version)
  1081. f = None
  1082. try:
  1083. f = open(distro_version, 'w')
  1084. f.write(self._getDistroStamp())
  1085. except IOError:
  1086. self.log.exception('failed to open file for writing: %s' %
  1087. distro_version)
  1088. raise FllError
  1089. finally:
  1090. if f:
  1091. f.close()
  1092. os.chmod(distro_version, 0444)
  1093. self._writeFile(arch, '/etc/hostname')
  1094. self._writeFile(arch, '/etc/hosts')
  1095. self._writeFile(arch, '/etc/motd.tail')
  1096. self.log.debug('writing final apt sources.list(s)')
  1097. self._writeAptLists(arch)
  1098. def _makeInitramfs(self, arch):
  1099. '''Generate the initramfs if update-initramfs was diverted'''
  1100. if '/usr/sbin/update-initramfs' in self.diverts and not self.opts.D:
  1101. chroot = os.path.join(self.temp, arch)
  1102. kvers = self._detectLinuxVersion(chroot)
  1103. for k in kvers:
  1104. cmd = 'update-initramfs -c -k %s' % k
  1105. if self.opts.v:
  1106. cmd += ' -v'
  1107. self._execInChroot(arch, cmd.split())
  1108. def _preseedDebconf(self, arch):
  1109. '''Preseed debconf with values read from package lists.'''
  1110. chroot = os.path.join(self.temp, arch)
  1111. debconf_list = self._getDebconfList(arch)
  1112. if debconf_list:
  1113. self.log.info('preseeding debconf in %s chroot...' % arch)
  1114. debconf = None
  1115. try:
  1116. debconf = open(os.path.join(chroot, 'tmp',
  1117. 'fll_debconf_selections'), 'w')
  1118. debconf.writelines([d + '\n' for d in debconf_list])
  1119. except IOError:
  1120. self.log.exception('failed to open file for writing: %s' %
  1121. '/tmp/fll_debconf_selections')
  1122. raise FllError
  1123. finally:
  1124. if debconf:
  1125. debconf.close()
  1126. cmd = 'debconf-set-selections '
  1127. if self.opts.v:
  1128. cmd += '--verbose '
  1129. cmd += '/tmp/fll_debconf_selections'
  1130. self._execInChroot(arch, cmd.split())
  1131. def _detectLinuxVersion(self, chroot):
  1132. '''Return version string of a singularly installed linux-image.'''
  1133. kvers = [f[f.find('-')+1:] for f in
  1134. os.listdir(os.path.join(chroot, 'boot'))
  1135. if f.startswith('vmlinuz-') or f.startswith('vmlinux-')]
  1136. if self.opts.D:
  1137. arch = chroot[chroot.rfind('/')+1:]
  1138. kvers = [f[len('linux-image-'):] for f in
  1139. self.pkgs[arch]['install']
  1140. if f.startswith('linux-image-')]
  1141. if len(kvers) > 0:
  1142. kvers.sort(cmp=apt_pkg.version_compare)
  1143. kvers.reverse()
  1144. return kvers
  1145. self.log.critical('failed to detect linux version installed in chroot')
  1146. raise FllError
  1147. def _detectLocalePkgs(self, i18n, wanted, cache):
  1148. '''Provide automated detection for extra i18n packages.'''
  1149. self.log.info('detecting i18n packages for %s...' % ' '.join(i18n))
  1150. i18n_module = ConfigObj('/usr/share/fll/data/locales-pkg-map')
  1151. self.log.debug('i18n_module:')
  1152. self.log.debug(i18n_module)
  1153. fll_locales = FllLocales(cache, wanted, i18n_module)
  1154. i18n_list = []
  1155. for locale in sorted(i18n):
  1156. try:
  1157. loc_pkg_list = fll_locales.detect_locale_packages(locale)
  1158. except FllLocalesError:
  1159. print_error('Failed to parse locale string: %s' % locale)
  1160. else:
  1161. i18n_list.extend(loc_pkg_list)
  1162. self.log.debug('i18n_list:')
  1163. self.log.debug(i18n_list)
  1164. return i18n_list
  1165. def _detectRecommendedPkgs(self, wanted, cache):
  1166. '''Provide automated detection for packages in recommends whitelist.'''
  1167. if self.conf['options']['apt_recommends'] == 'yes':
  1168. return []
  1169. self.log.info('detecting whitelisted recommended packages...')
  1170. rec_module = ConfigObj(os.path.join(self.opts.s, 'packages',
  1171. 'packages.d', 'recommends'))
  1172. try:
  1173. rec_dict = dict([(p, True) for p in
  1174. self.__lines2list(rec_module['packages'])])
  1175. except KeyError:
  1176. self.log.debug('rec_dict:')
  1177. return []
  1178. self.log.debug('rec_dict:')
  1179. self.log.debug(rec_dict)
  1180. rec_list = []
  1181. for p in wanted.keys():
  1182. if not p in rec_dict:
  1183. continue
  1184. package = cache[p]
  1185. current = package.current_ver
  1186. if not current:
  1187. versions = package.version_list
  1188. if not versions:
  1189. continue
  1190. version = versions[0]
  1191. for other_version in versions:
  1192. if apt_pkg.version_compare(version.ver_str,
  1193. other_version.ver_str) < 0:
  1194. version = other_version
  1195. current = version
  1196. depends = current.depends_list
  1197. list = depends.get('Recommends', [])
  1198. for dependency in list:
  1199. depdone = 0
  1200. for deppart in dependency:
  1201. if depdone > 0:
  1202. continue
  1203. name = deppart.target_pkg.name
  1204. if name in wanted.keys():
  1205. depdone = 1
  1206. continue
  1207. if depdone == 0:
  1208. name = dependency[0].target_pkg.name
  1209. dep = cache[name]
  1210. if dep.current_ver:
  1211. continue
  1212. rec_list.append(dep.name)
  1213. self.log.debug('rec_list:')
  1214. self.log.debug(rec_list)
  1215. return rec_list
  1216. def __getSourcePkg(self, pkg, depcache, records):
  1217. '''Get the source package name of a given package.'''
  1218. version = depcache.get_candidate_ver(pkg)
  1219. if not version:
  1220. return None
  1221. file, index = version.file_list.pop(0)
  1222. records.lookup((file, index))
  1223. srcpkg = pkg.name
  1224. srcver = version.ver_str
  1225. if records.source_pkg != "":
  1226. srcpkg = records.source_pkg
  1227. if records.source_ver != "":
  1228. srcver = records.source_ver
  1229. return [srcpkg,srcver]
  1230. def _collectManifest(self, arch):
  1231. '''Collect package and source package URI information from each
  1232. chroot.'''
  1233. chroot = os.path.join(self.temp, arch)
  1234. self.log.info('collecting package manifest for %s...' % arch)
  1235. cache = apt_pkg.Cache(apt.progress.base.OpProgress())
  1236. records = apt_pkg.PackageRecords(cache)
  1237. depcache = apt_pkg.DepCache(cache)
  1238. manifest = dict([(p.name, p.current_ver.ver_str)
  1239. for p in cache.packages if p.current_ver
  1240. and not p.name.startswith('cdebootstrap-helper')])
  1241. if 'install' in self.pkgs[arch]:
  1242. manifest.update(self.pkgs[arch]['install'])
  1243. if 'langpack' in self.pkgs[arch]:
  1244. manifest.update(self.pkgs[arch]['langpack'])
  1245. if 'extras' in self.pkgs[arch]:
  1246. manifest.update(self.pkgs[arch]['extras'])
  1247. self.pkgs[arch]['manifest'] = manifest
  1248. if self.opts.B:
  1249. return
  1250. self.log.info('querying source package URIs for %s...' % arch)
  1251. packages = manifest.keys()
  1252. packages.sort()
  1253. srcpkg_seen = dict()
  1254. uris = []
  1255. for p in packages:
  1256. for k in self._detectLinuxVersion(chroot):
  1257. if p.endswith('-modules-' + k):
  1258. p = p[:p.find('-modules-' + k)]
  1259. p += '-source'
  1260. if p.startswith('cdebootstrap-helper'):
  1261. continue
  1262. srcpv = self.__getSourcePkg(cache[p], depcache, records)
  1263. srcpkg=srcpv[0]
  1264. srcver=srcpv[1]
  1265. if not srcpkg:
  1266. self.log.critical('failed to lookup srcpkg name for %s' % p)
  1267. raise FllError
  1268. self.log.debug('%s -> %s' % (p, srcpkg))
  1269. if srcpkg in srcpkg_seen:
  1270. self.log.debug('already processed %s, skipping...' % srcpkg)
  1271. continue
  1272. else:
  1273. srcpkg_seen[srcpkg] = True
  1274. u = []
  1275. sources = apt_pkg.SourceRecords()
  1276. sources.restart()
  1277. while sources.lookup(srcpkg):
  1278. if sources.version == srcver:
  1279. u.extend([sources.index.archive_uri(sources.files[f][2])
  1280. for f in range(len(sources.files))])
  1281. if len(u) > 0:
  1282. self.log.debug(u)
  1283. uris.extend(u)
  1284. else:
  1285. self.log.critical('')
  1286. self.log.critical('----------------------------------')
  1287. self.log.critical('failed to query source uris for %s' % srcpkg)
  1288. self.log.critical('----------------------------------')
  1289. self.log.critical('')
  1290. # raise FllError
  1291. uris.sort()
  1292. self.pkgs[arch]['source'] = uris
  1293. def _installPkgs(self, arch, codename):
  1294. '''Install packages.'''
  1295. cache = apt_pkg.Cache(apt.progress.base.OpProgress())
  1296. if cache.ver_file_count < 200:
  1297. self.log.info('installPkgs cache had %s' % cache.ver_file_count)
  1298. cache = apt_pkg.Cache(apt.progress.base.OpProgress())
  1299. self.log.info('installPkgs cache has %s' % cache.ver_file_count)
  1300. i18n_list = self.__lines2list(self.conf['packages']['i18n'])
  1301. pkgs_base = [p.name for p in cache.packages if p.current_ver]
  1302. pkgs_want = self.__filterList(pkgs_base + self._getPackageList(arch))
  1303. pkgs_dict = dict([(p, True) for p in pkgs_want])
  1304. pkgs_want = self.__filterList(pkgs_dict.keys()
  1305. + self._detectLocalePkgs(i18n_list, pkgs_dict, cache)
  1306. + self._detectRecommendedPkgs(pkgs_dict, cache))
  1307. pkgs_had = list()
  1308. if self.opts.D:
  1309. self.pkgs[arch]['install'] = dict()
  1310. self.log.info('downloading packages in %s chroot...' % arch)
  1311. # re-runs detect*Pkgs using what will now be installed
  1312. # repeat until nothing was added
  1313. while len(pkgs_want) > len(pkgs_had):
  1314. self.log.debug('%i new in package list for this run'
  1315. % (len(pkgs_want)-len(pkgs_had)))
  1316. pkgs_had = list()
  1317. pkgs_had.extend(pkgs_want)
  1318. self._aptGetInstall(arch, self.__filterList(pkgs_want),
  1319. download_only = True)
  1320. chroot = os.path.join(self.temp, arch)
  1321. aptcache = os.path.join(chroot, 'var/cache/apt/archives/*.deb')
  1322. for debfile in glob.glob(aptcache):
  1323. pkg, vers, extra = debfile.split('/')[-1].split('_')
  1324. # big dict given below to detect*Pkgs
  1325. pkgs_dict[pkg] = True
  1326. if self.opts.D:
  1327. # create dict with package name = version to extend manifest
  1328. self.pkgs[arch]['install'][pkg] = vers.replace('%3a', ':')
  1329. for pkg in ( self._detectLocalePkgs(i18n_list, pkgs_dict, cache)
  1330. + self._detectRecommendedPkgs(pkgs_dict, cache) ):
  1331. if pkg not in pkgs_want:
  1332. pkgs_want.append(pkg)
  1333. self.log.debug('%s now added to wanted packages had %i now %i' % (pkg,len(pkgs_had),len(pkgs_want)))
  1334. if not self.opts.D:
  1335. self.log.info('installing packages in %s chroot...' % arch)
  1336. self._aptGetInstall(arch, pkgs_want)
  1337. # Fetch extra debs if appropriate and reprepro them
  1338. extras = []
  1339. commextra = []
  1340. efitypes = { 'x86_64-efi':'amd64','i386-efi':'ia32' }
  1341. for efitype in (efitypes.keys()):
  1342. archextra = []
  1343. if os.path.isdir(os.path.join(self.temp, arch, 'usr/lib/grub/%s' % efitype)):
  1344. commextra.append('grub-efi')
  1345. archextra.append('grub-efi-%s' % efitypes[efitype])
  1346. extras.append(archextra)
  1347. if len(extras):
  1348. commextra.append('grub2-fll-portable-efi')
  1349. extras.append(commextra)
  1350. self.pkgs[arch]['extras'] = dict()
  1351. self._execInChroot(arch, ['apt-get', 'clean'])
  1352. for archextra in extras:
  1353. self._aptGetInstall(arch, archextra,
  1354. download_only = True)
  1355. # Generate a basic reprepro conf/distributions.
  1356. extras = os.path.join(self.temp, arch, 'fll', 'extras')
  1357. extras_conf = os.path.join(extras, 'conf')
  1358. if not os.path.isdir(extras_conf):
  1359. os.makedirs(extras_conf)
  1360. extras_dist = os.path.join(extras, 'conf', 'distributions')
  1361. rconf = None
  1362. try:
  1363. rconf = open(extras_dist, "w")
  1364. rconf.write('Codename: %s\n' % codename)
  1365. rconf.write('Architectures: ')
  1366. for a in self.conf['archs'].keys():
  1367. rconf.write(''.join([a,' ']))
  1368. rconf.write('\n')
  1369. rconf.write('Components: main\n')
  1370. rconf.write('Description: extra packages\n')
  1371. except IOError:
  1372. self.log.exception('error preparing reprepro for extras')
  1373. raise FllError
  1374. finally:
  1375. if rconf:
  1376. rconf.close()
  1377. # Find all the debs and includedeb them.
  1378. chroot = os.path.join(self.temp, arch)
  1379. aptcache = os.path.join(chroot, 'var/cache/apt/archives/*.deb')
  1380. for debfile in glob.glob(aptcache):
  1381. if not self.opts.D:
  1382. self._execCmd(['reprepro', '-Vb', extras, 'includedeb', codename,
  1383. debfile])
  1384. # create dict with package name = version to extend manifest
  1385. pkg, vers, extra = debfile.split('/')[-1].split('_')
  1386. self.pkgs[arch]['extras'][pkg] = vers.replace('%3a', ':')
  1387. # Calculate packages for each language.
  1388. self.pkgs[arch]['langpack'] = dict()
  1389. if 'lang' not in self.conf['packages']:
  1390. return
  1391. lang_list = self.__lines2list(self.conf['packages']['lang'])
  1392. lang_full = pkgs_want
  1393. i18n = os.path.join(self.temp, 'staging', 'i18n')
  1394. for lang in lang_list:
  1395. lang_pkgs = self._detectLocalePkgs([ lang ], pkgs_dict, cache)
  1396. i18n_arch = os.path.join(i18n, arch)
  1397. if not os.path.isdir(i18n_arch):
  1398. os.makedirs(i18n_arch)
  1399. i18n_lang = os.path.join(i18n, arch, lang)
  1400. i18nlist = None
  1401. try:
  1402. i18nlist = open(i18n_lang, "w")
  1403. for pkg in lang_pkgs:
  1404. i18nlist.write('%s ' % (pkg))
  1405. except IOError:
  1406. self.log.exception('error writing i18n file for lang: %s' %
  1407. lang)
  1408. raise FllError
  1409. finally:
  1410. if i18nlist:
  1411. i18nlist.close()
  1412. lang_full.extend(lang_pkgs)
  1413. # Fetch all extra lang packages and reprepro them.
  1414. if lang_pkgs:
  1415. self._execInChroot(arch, ['apt-get', 'clean'])
  1416. self._aptGetInstall(arch, self.__filterList(lang_full),
  1417. download_only = True)
  1418. # Generate a basic reprepro conf/distributions.
  1419. i18n_conf = os.path.join(i18n, 'conf')
  1420. if not os.path.isdir(i18n_conf):
  1421. os.mkdir(i18n_conf)
  1422. i18n_dist = os.path.join(i18n, 'conf', 'distributions')
  1423. rconf = None
  1424. try:
  1425. rconf = open(i18n_dist, "w")
  1426. rconf.write('Codename: %s\n' % codename)
  1427. rconf.write('Architectures: ')
  1428. for a in self.conf['archs'].keys():
  1429. rconf.write(''.join([a,' ']))
  1430. rconf.write('\n')
  1431. rconf.write('Components: main\n')
  1432. rconf.write('Description: i18n packages\n')
  1433. except IOError:
  1434. self.log.exception('error preparing reprepro')
  1435. raise FllError
  1436. finally:
  1437. if rconf:
  1438. rconf.close()
  1439. # Find all the debs and includedeb them.
  1440. chroot = os.path.join(self.temp, arch)
  1441. aptcache = os.path.join(chroot, 'var/cache/apt/archives/*.deb')
  1442. for debfile in glob.glob(aptcache):
  1443. if not self.opts.D:
  1444. self._execCmd(['reprepro', '-Vb', i18n, 'includedeb', codename,
  1445. debfile])
  1446. # create dict with package name = version to extend manifest
  1447. pkg, vers, extra = debfile.split('/')[-1].split('_')
  1448. self.pkgs[arch]['langpack'][pkg] = vers.replace('%3a', ':')
  1449. def _postInst(self, arch):
  1450. '''Run package module postinst scripts in a chroot.'''
  1451. if self.opts.D:
  1452. return
  1453. chroot = os.path.join(self.temp, arch)
  1454. self.log.info('performing post-install tasks in %s chroot...' % arch)
  1455. for script in self._getPostinstList(arch):
  1456. sname = os.path.basename(script)
  1457. try:
  1458. shutil.copy(script, os.path.join(chroot, 'tmp'))
  1459. os.chmod(os.path.join(chroot, 'tmp', sname), 0755)
  1460. except:
  1461. self.log.exception('error preparing postinst script: %s' %
  1462. sname)
  1463. raise FllError
  1464. cmd = '/tmp/%s postinst' % sname
  1465. self._execInChroot(arch, cmd.split())
  1466. os.unlink(os.path.join(chroot, 'tmp', sname))
  1467. def _zerologs(self, arch, dir, fnames):
  1468. '''Truncate all log files.'''
  1469. chroot = os.path.join(self.temp, arch)
  1470. chrootdir = dir.partition(chroot)[2]
  1471. for f in fnames:
  1472. if not os.path.isfile(os.path.join(dir, f)):
  1473. continue
  1474. self._writeFile(arch, os.path.join(chrootdir, f))
  1475. def _cleanChroot(self, arch):
  1476. '''Remove unwanted content from a chroot.'''
  1477. self.log.info('purging unwanted content from %s chroot...' % arch)
  1478. chroot = os.path.join(self.temp, arch)
  1479. cmd = 'dpkg --purge fll-live-initramfs'
  1480. self._execInChroot(arch, cmd.split())
  1481. self._execInChroot(arch, 'apt-get clean'.split())
  1482. self._execInChroot(arch, 'dpkg --clear-avail'.split())
  1483. self._execInChroot(arch, 'del-gpg-helpers'.split())
  1484. os.path.walk(os.path.join(chroot, 'var/log'), self._zerologs, arch)
  1485. def _chrootSquashfs(self, arch):
  1486. '''Make squashfs filesystem image of chroot.'''
  1487. self.log.info('creating squashfs filesystem of %s chroot...' % arch)
  1488. chroot = os.path.join(self.temp, arch)
  1489. image_file = self._getDistroImageFile(arch)
  1490. if self.opts.D:
  1491. cmd = ['touch', image_file]
  1492. self._execInChroot(arch, cmd)
  1493. return
  1494. cmd = ['mksquashfs', '.', image_file, '-noappend']
  1495. exclude_file = os.path.join(self.opts.s, 'data', 'fll_sqfs_exclusion')
  1496. shutil.copy(exclude_file, os.path.join(chroot, 'tmp'))
  1497. cmd.extend(['-wildcards', '-ef', '/tmp/fll_sqfs_exclusion'])
  1498. # set compression algorithm for squashfs-tools >= 4.1
  1499. squashfs_comp = self.conf['options'].get('squashfs_comp')
  1500. if squashfs_comp in ['gzip', 'lzo', 'xz']:
  1501. self.log.info('using squashfs(%s)...' % squashfs_comp)
  1502. cmd.extend(['-comp', '%s' % squashfs_comp])
  1503. if squashfs_comp == 'xz':
  1504. if arch == 'amd64' or arch == 'i386':
  1505. cmd.extend(['-Xbcj', 'x86'])
  1506. elif arch == 'powerpc' or arch == 'ppc64':
  1507. cmd.extend(['-Xbcj', 'powerpc'])
  1508. if self.opts.l or self.opts.q:
  1509. cmd.append('-no-progress')
  1510. # can only be last argument
  1511. cmd.extend(['-e', image_file])
  1512. self._execInChroot(arch, cmd)
  1513. def secondSquashfs(self):
  1514. '''Make squashfs filesystem image of non chroot other material.'''
  1515. if self.opts.D:
  1516. return
  1517. self.log.info('creating secondary squashfs filesystem')
  1518. stage = os.path.join(self.temp, 'staging')
  1519. sqfs_file = os.path.join(stage, self.conf['distro']['FLL_IMAGE_DIR'], '%s.2' % self.conf['distro']['FLL_IMAGE_FILE'])
  1520. excl_file = os.path.join(stage, self.conf['distro']['FLL_IMAGE_DIR'], '%s.2.excl' % self.conf['distro']['FLL_IMAGE_FILE'])
  1521. excl = open(excl_file, 'w')
  1522. excl.write('autorun.inf\n')
  1523. excl.write('boot\n')
  1524. excl.write('boot/*\n')
  1525. excl.write('boot.catalog\n')
  1526. excl.write('cdrom.ico\n')
  1527. excl.write('efi\n')
  1528. excl.write('efi/*\n')
  1529. excl.write('efi.img\n')
  1530. excl.write('iso_uuid\n')
  1531. excl.write('%s\n' % self.conf['distro']['FLL_IMAGE_DIR'])
  1532. excl.write('%s/*\n' % self.conf['distro']['FLL_IMAGE_DIR'])
  1533. excl.close()
  1534. cmd = ['mksquashfs', stage, sqfs_file, '-noappend']
  1535. cmd.extend(['-wildcards', '-ef', excl_file])
  1536. # set compression algorithm for squashfs-tools >= 4.1
  1537. squashfs_comp = self.conf['options'].get('squashfs_comp')
  1538. if squashfs_comp in ['gzip', 'lzo', 'xz']:
  1539. self.log.info('using squashfs(%s)...' % squashfs_comp)
  1540. cmd.extend(['-comp', '%s' % squashfs_comp])
  1541. if squashfs_comp == 'xz':
  1542. arch = self.conf['archs'].keys()[0]
  1543. if arch == 'amd64' or arch == 'i386':
  1544. cmd.extend(['-Xbcj', 'x86'])
  1545. elif arch == 'powerpc' or arch == 'ppc64':
  1546. cmd.extend(['-Xbcj', 'powerpc'])
  1547. if self.opts.l or self.opts.q:
  1548. cmd.append('-no-progress')
  1549. self._execCmd(cmd)
  1550. self._nuke(os.path.join(stage,'extras'))
  1551. self._nuke(os.path.join(stage,'i18n'))
  1552. os.unlink(excl_file)
  1553. def _stageArch(self, arch):
  1554. '''Stage files for an arch for final genisofs.'''
  1555. self.log.info('staging live %s media...' % arch)
  1556. chroot = os.path.join(self.temp, arch)
  1557. boot_dir = os.path.join(self.temp, 'staging', 'boot')
  1558. image_file = os.path.join(chroot, self._getDistroImageFile(arch))
  1559. image_dir = os.path.join(self.temp, 'staging',
  1560. self.conf['distro']['FLL_IMAGE_DIR'])
  1561. try:
  1562. os.chmod(image_file, 0644)
  1563. shutil.move(image_file, image_dir)
  1564. except IOError:
  1565. self.log.exception('failed to move squashfs image to staging dir')
  1566. raise FllError
  1567. if self.opts.D:
  1568. return
  1569. kvers = self._detectLinuxVersion(chroot)
  1570. for k in kvers:
  1571. initrd = os.path.join(chroot, 'boot', 'initrd.img-' + k)
  1572. if os.path.isfile(initrd):
  1573. self.log.debug('copying %s to staging dir' % initrd)
  1574. shutil.copy(initrd, boot_dir)
  1575. else:
  1576. self.log.critical('could not find initramfs image to ' +
  1577. 'copy to staging dir.')
  1578. raise FllError
  1579. images = glob.glob(os.path.join(chroot, 'boot', 'vmlinu*-' + k))
  1580. if len(images) == 1:
  1581. self.log.debug('copying %s to staging dir' % images[0])
  1582. shutil.copy(images[0], boot_dir)
  1583. else:
  1584. self.log.critical('could not find linux kernel image to ' +
  1585. 'copy to staging dir.')
  1586. raise FllError
  1587. if os.path.isdir(os.path.join(chroot, 'usr/lib/grub')):
  1588. data_dir = os.path.join(self.opts.s, 'data')
  1589. grub_dir = os.path.join(boot_dir, 'grub')
  1590. if not os.path.isdir(grub_dir):
  1591. os.makedirs(grub_dir, 0755)
  1592. if not os.path.isfile(os.path.join(grub_dir, 'grub.cfg')):
  1593. shutil.copy(os.path.join(self.opts.s, 'data/grub.cfg'), grub_dir)
  1594. shutil.copy(os.path.join(chroot, 'usr/share/grub/unicode.pf2'), grub_dir)
  1595. shutil.copytree(os.path.join(self.opts.s, 'data/locales'), os.path.join(grub_dir, 'locales'))
  1596. shutil.copytree(os.path.join(self.opts.s, 'data/tz'), os.path.join(grub_dir, 'tz'))
  1597. theme_dir = 'themes/%s' % self.conf['distro']['FLL_GFXBOOT_THEME']
  1598. if os.path.isfile(os.path.join(chroot, 'usr/share/grub/%s/theme.txt' % theme_dir)):
  1599. shutil.copytree(os.path.join(chroot, 'usr/share/grub/%s' % theme_dir), os.path.join(grub_dir, theme_dir))
  1600. gfile_dir = glob.glob(os.path.join(chroot, 'usr/lib/grub/*-pc'))[0]
  1601. grub2_modules = glob.glob(os.path.join(gfile_dir, '*.mod'))
  1602. if len(grub2_modules) > 0:
  1603. gfiles = [os.path.join(gfile_dir, f) for f in os.listdir(gfile_dir)
  1604. if f.endswith('.mod') or f.endswith('.img')
  1605. or f.endswith('.lst')]
  1606. gfiles.append(os.path.join(chroot, 'tmp/grub_eltorito'))
  1607. else:
  1608. gfiles = [os.path.join(gfile_dir, f) for f in os.listdir(gfile_dir)
  1609. if f.startswith('stage2') or f.startswith('iso9660')]
  1610. if len(gfiles) > 0:
  1611. self.log.debug('copying grub stage files to boot dir')
  1612. grub_dir = os.path.join(boot_dir, 'grub', 'i386-pc')
  1613. if not os.path.isdir(grub_dir):
  1614. os.makedirs(grub_dir, 0755)
  1615. else:
  1616. self.log.exception('grub stage files not found')
  1617. raise FllError
  1618. for file in gfiles:
  1619. try:
  1620. shutil.copy(file, grub_dir)
  1621. except IOError:
  1622. self.log.exception('failed to copy grub file to staging dir')
  1623. raise FllError
  1624. # efi
  1625. efitypes = { 'x86_64-efi':'bootx64','i386-efi':'bootia32' }
  1626. have_efi = ""
  1627. for efitype in (efitypes.keys()):
  1628. gfile_dir = os.path.join(chroot, 'usr/lib/grub/%s' % efitype)
  1629. if os.path.isdir(gfile_dir):
  1630. have_efi = "true"
  1631. self.log.info('We found efi related files and set have_efi true')
  1632. gfiles = [os.path.join(gfile_dir, f) for f in os.listdir(gfile_dir)
  1633. if f.endswith('.mod') or f.endswith('.lst')]
  1634. gfiles.append(os.path.join(chroot,'/usr/share/grub/unicode.pf2'))
  1635. if len(gfiles) > 0:
  1636. self.log.info('copying grub %s stage files to boot dir' % efitype)
  1637. grub_dir = os.path.join(boot_dir, 'grub', efitype)
  1638. if not os.path.isdir(grub_dir):
  1639. os.makedirs(grub_dir, 0755)
  1640. for file in gfiles:
  1641. try:
  1642. shutil.copy(file, grub_dir)
  1643. except IOError:
  1644. self.log.exception('failed to copy grub efi file to staging dir')
  1645. raise FllError
  1646. efi_dir = os.path.join(self.temp, 'staging/efi/boot')
  1647. if not os.path.isdir(efi_dir):
  1648. os.makedirs(efi_dir, 0755)
  1649. try:
  1650. shutil.copy(os.path.join(chroot, 'tmp/efi/boot/%s.efi' % efitypes[efitype]), efi_dir)
  1651. except IOError:
  1652. self.log.exception('failed to copy efi %s boot files to staging dir' % efitype)
  1653. raise FllError
  1654. if have_efi != "" and not os.path.isfile(os.path.join(self.temp, 'staging', 'efi.img')):
  1655. try:
  1656. self.log.info('Creating efi.img using mformat and mcopy.')
  1657. cmd = [ 'mformat', '-C', '-f', '2880', '-L', '16', '-i', '/tmp/efi.img', '::' ]
  1658. self._execInChroot(arch, cmd)
  1659. cmd = [ 'mcopy', '-s', '-i', '/tmp/efi.img', '/tmp/efi_img/efi', '::/' ]
  1660. self._execInChroot(arch, cmd)
  1661. shutil.copy(os.path.join(chroot, 'tmp/efi.img'), os.path.join(self.temp, 'staging'))
  1662. shutil.copy(os.path.join(chroot, 'tmp/iso_uuid'), os.path.join(self.temp, 'staging'))
  1663. uuid_dir = os.path.join(chroot, 'tmp/uuid')
  1664. ufiles = [os.path.join(uuid_dir, f) for f in os.listdir(uuid_dir)]
  1665. for file in ufiles:
  1666. shutil.copy(file, os.path.join(boot_dir, 'grub'))
  1667. except IOError:
  1668. self.log.exception('failed to create efi.img')
  1669. raise FllError
  1670. memtest = os.path.join(chroot, 'boot', 'memtest86+.bin')
  1671. memtest_out = os.path.join(boot_dir, 'memtest')
  1672. if os.path.isfile(memtest):
  1673. self.log.debug('copying memtest86+ to boot dir')
  1674. try:
  1675. shutil.copy(memtest, memtest_out)
  1676. except IOError:
  1677. self.log.exception('failed to copy memtest86+ to staging dir')
  1678. raise FllError
  1679. def _writeGrubCfg(self, stage_dir, boot_dir, grub_dir, kvers,
  1680. timeout, cmdline):
  1681. '''Write grub.cfg for live media.'''
  1682. self.log.debug('writing grub.cfg for live media')
  1683. grubcfg = open(os.path.join(grub_dir, 'kernels.cfg'), 'w')
  1684. distro = self.conf['distro']['FLL_DISTRO_NAME']
  1685. kcount = { '686': 0, 'amd': 0, 'pae': 0, '486': 0, 'ppc': 0 }
  1686. for k in kvers:
  1687. cpu = k[k.rfind('-') + 1:]
  1688. vmlinuz = 'vmlinuz-%s' % k
  1689. initrd = 'initrd.img-%s' % k
  1690. display_iso = 'From CD/DVD/ISO: %s.%s %s' % (distro, cpu, self.conf['distro']['FLL_FLAVOUR'])
  1691. display_hdd = 'From Stick/HDD: %s.%s %s' % (distro, cpu, self.conf['distro']['FLL_FLAVOUR'])
  1692. for f in [vmlinuz, initrd]:
  1693. if not os.path.isfile(os.path.join(boot_dir, f)):
  1694. self.log.critical('%s was not found in %s' % (f, boot_dir))
  1695. raise FllError
  1696. if cpu[0:3] == 'amd':
  1697. grubcfg.write('if cpuid -l; then\n')
  1698. grubcfg.write('havekernel="Y"\n')
  1699. grubcfg.write('title=""\n')
  1700. grubcfg.write('for kopt in %s $kopts %s boot=fll quiet systemd.show_status=1; do\n' % (vmlinuz[8:], cmdline))
  1701. grubcfg.write(' if [ -n "$title" ] ; then\n')
  1702. grubcfg.write(' title="$title $kopt";\n')
  1703. grubcfg.write(' else\n')
  1704. grubcfg.write(' title="$kopt";\n')
  1705. grubcfg.write(' fi;\n')
  1706. grubcfg.write('done\n')
  1707. grubcfg.write('menuentry \"%s \" --class=%s.%s \"$title\" {' % (display_iso, distro, cpu))
  1708. grubcfg.write('# set arguments above with the editor\n')
  1709. grubcfg.write('linux /boot/vmlinuz-$2\n')
  1710. grubcfg.write('initrd /boot/%s\n' % initrd)
  1711. grubcfg.write('}\n')
  1712. grubcfg.write('menuentry \"%s \" --class=%s.%s \"$title fromhd\" {' % (display_hdd, distro, cpu))
  1713. grubcfg.write('# set arguments above with the editor\n')
  1714. grubcfg.write('linux /boot/vmlinuz-$2\n')
  1715. grubcfg.write('initrd /boot/%s\n' % initrd)
  1716. grubcfg.write('}\n')
  1717. if cpu[0:3] == 'amd':
  1718. grubcfg.write('fi\n')
  1719. kcount[cpu[0:3]] = kcount[cpu[0:3]] + 1
  1720. grubcfg.write('if [ "${havekernel}" != "Y" ]; then\n')
  1721. grubcfg.write(' menuentry --class=find.none "NO SUITABLE KERNELS AVAILABLE" {echo $@')
  1722. grubcfg.write(' echo "There are no kernels suitable for this machine available."\n')
  1723. grubcfg.write(' echo ""\n')
  1724. grubcfg.write(' if ! cpuid -l; then\n')
  1725. grubcfg.write(' echo "This machine is NOT 64bit capable."\n')
  1726. grubcfg.write(' for kk in /boot/vmlinu*.*86; do\n')
  1727. grubcfg.write(' if [ "$kk" != "/boot/vmlinu*.*86" ]; then\n')
  1728. grubcfg.write(' have32="true"\n')
  1729. grubcfg.write(' fi\n')
  1730. grubcfg.write(' done\n')
  1731. grubcfg.write(' if [ "${have32}" != "true" ]; then\n')
  1732. grubcfg.write(' echo "There are no 32bit kernels available"\n')
  1733. grubcfg.write(' echo "It appears you are trying to boot a 64bit release on a 32bit machine"\n')
  1734. grubcfg.write(' echo "This cannot work!"\n')
  1735. grubcfg.write(' fi\n')
  1736. grubcfg.write(' fi\n')
  1737. grubcfg.write(' echo "Press Escape to return to the main menu"\n')
  1738. grubcfg.write(' sleep --interruptible 9999\n')
  1739. grubcfg.write(' menu_reload\n')
  1740. grubcfg.write(' }\n')
  1741. grubcfg.write('fi\n')
  1742. grubcfg.close()
  1743. self.log.debug('writing loopback.cfg for live media')
  1744. grubcfg = open(os.path.join(grub_dir, 'loopback.cfg'), 'w')
  1745. grubcfg.write('source /boot/grub/grub.cfg\n')
  1746. grubcfg.close()
  1747. self.log.debug('writing variable.cfg for live media')
  1748. grubcfg = open(os.path.join(grub_dir, 'variable.cfg'), 'w')
  1749. grub_theme = 'grub/themes/%s/theme.txt' % self.conf['distro']['FLL_GFXBOOT_THEME']
  1750. if os.path.isfile(os.path.join(boot_dir, grub_theme)):
  1751. grubcfg.write('grub_theme=/boot/%s\n' % grub_theme)
  1752. grubcfg.write('timeout=%s\n' % timeout)
  1753. grubcfg.close()
  1754. def _configBootKvers(self,stage_dir):
  1755. kvers = self._detectLinuxVersion(stage_dir)
  1756. if len(kvers) < 1:
  1757. self.log.critical('failed to find linux kernel image to include in boot conf')
  1758. raise FllError
  1759. return kvers
  1760. def _configBootTimeout(self):
  1761. timeout = self.conf['options'].get('boot_timeout')
  1762. if not timeout:
  1763. timeout = '-1'
  1764. return timeout
  1765. def _configBootCmdline(self):
  1766. cmdline = self.conf['options'].get('boot_cmdline')
  1767. if not cmdline:
  1768. cmdline = ''
  1769. return cmdline
  1770. def _writeGrubWrap(self):
  1771. '''Write final GRUB configuration for live media.'''
  1772. stage_dir = os.path.join(self.temp, 'staging')
  1773. boot_dir = os.path.join(stage_dir, 'boot')
  1774. grub_dir = os.path.join(boot_dir, 'grub')
  1775. if not os.path.isdir(grub_dir):
  1776. return
  1777. self.log.debug('writing grub config for live media')
  1778. kvers = self._configBootKvers(stage_dir)
  1779. timeout = self._configBootTimeout()
  1780. cmdline = self._configBootCmdline()
  1781. self._writeGrubCfg(stage_dir, boot_dir, grub_dir, kvers,
  1782. timeout, cmdline)
  1783. def writeBootldr(self):
  1784. '''Choose which bootloader to configure based on archs, and
  1785. call the appropriate method to do that.'''
  1786. archs = self.conf['archs'].keys()
  1787. self._writeGrubWrap()
  1788. if self.opts.D:
  1789. return
  1790. def __md5sum(self, filename):
  1791. '''Calculate md5sum of a file and return it.'''
  1792. return apt_pkg.md5sum(file(filename))
  1793. def __sha256sum(self, filename):
  1794. '''Calculate sha256sum of a file and return it.'''
  1795. return apt_pkg.sha256sum(file(filename))
  1796. def _md5sums(self, base, dir, fnames):
  1797. '''Function given to os.path.walk of self.writeMd5Sums().'''
  1798. for f in fnames:
  1799. file = os.path.join(dir, f)
  1800. filename = file.partition(base)[2].lstrip('/')
  1801. if not os.path.isfile(file) or f == 'md5sums':
  1802. continue
  1803. if dir.endswith('grub') and f.find('stage') >= 0:
  1804. continue
  1805. if f.find('grub_eltorito') >= 0:
  1806. continue
  1807. if f.find('iso_uuid') >= 0:
  1808. continue
  1809. md5sums = None
  1810. try:
  1811. md5sums = open(os.path.join(base, 'md5sums'), 'a')
  1812. md5sums.write("%s *%s\n" % (self.__md5sum(file), filename))
  1813. except IOError:
  1814. self.log.exception('failed to write md5sums file')
  1815. raise FllError
  1816. finally:
  1817. if md5sums:
  1818. md5sums.close()
  1819. def writeMd5Sums(self):
  1820. '''Calculate md5sums of major release contents.'''
  1821. self.log.info('calculating md5sums of live media...')
  1822. stage = os.path.join(self.temp, 'staging')
  1823. os.path.walk(stage, self._md5sums, stage)
  1824. def _signFile(self, file):
  1825. '''Sign a file with hashkey if available.'''
  1826. if self.opts.k:
  1827. self.log.info('Signing file %s...' % file)
  1828. cmd = ['gpg', '-s', '--default-key']
  1829. cmd.append(self.opts.k)
  1830. cmd.append(file)
  1831. self._execCmd(cmd)
  1832. else:
  1833. self.log.info('Not signing file %s: No key given.' % file)
  1834. def __archManifest(self, arch):
  1835. '''Write manifest information to file.'''
  1836. pkgs = self.pkgs[arch]['manifest'].keys()
  1837. pkgs.sort(key=len)
  1838. l = len(pkgs[-1])
  1839. pkgs.sort()
  1840. return ["%s %s\n" % (p.ljust(l), self.pkgs[arch]['manifest'][p])
  1841. for p in pkgs]
  1842. def _writeManifests(self, file):
  1843. '''Write package manifest lists.'''
  1844. archs = self.conf['archs'].keys()
  1845. for arch in archs:
  1846. manifest_name = '%s.%s.manifest' % (file, arch)
  1847. manifest_file = os.path.join(self.opts.o, manifest_name)
  1848. manifest = None
  1849. try:
  1850. manifest = open(manifest_file, 'w')
  1851. manifest.writelines(self.__archManifest(arch))
  1852. except IOError:
  1853. self.log.exception('failed to write file: %s' % manifest_file)
  1854. raise FllError
  1855. finally:
  1856. if manifest:
  1857. manifest.close()
  1858. os.chown(manifest_file, self.opts.u, self.opts.g)
  1859. def _writeSources(self, file):
  1860. '''Write source URI lists.'''
  1861. sources_list = []
  1862. archs = self.conf['archs'].keys()
  1863. for arch in archs:
  1864. sources_list.extend(self.pkgs[arch]['source'])
  1865. sources_list = self.__filterList(sources_list, dup_warn = False)
  1866. sources_name = file + '.sources'
  1867. sources_file = os.path.join(self.opts.o, sources_name)
  1868. sources = None
  1869. try:
  1870. sources = open(sources_file, 'w')
  1871. sources.writelines(["%s\n" % s for s in sources_list])
  1872. except IOError:
  1873. self.log.exception('failed to write file: %s' % sources_file)
  1874. raise FllError
  1875. finally:
  1876. if sources:
  1877. sources.close()
  1878. os.chown(sources_file, self.opts.u, self.opts.g)
  1879. cached = dict()
  1880. for r in self.conf['repos']:
  1881. if self.conf['repos'][r].get('cached'):
  1882. cached_uri = self.conf['repos'][r]['cached']
  1883. uri = self.conf['repos'][r]['uri']
  1884. cached[cached_uri.rstrip('/')] = uri.rstrip('/')
  1885. if len(cached.keys()) > 0:
  1886. os.rename(sources_file, sources_file + '-cached')
  1887. else:
  1888. return
  1889. sources = None
  1890. try:
  1891. sources = open(sources_file, 'w')
  1892. for s in sources_list:
  1893. for c in cached.keys():
  1894. if s.startswith(c):
  1895. s = s.replace(c, cached[c], 1)
  1896. break
  1897. sources.write('%s\n' % s)
  1898. except IOError:
  1899. self.log.exception('failed to write file: %s' % sources_file)
  1900. raise FllError
  1901. finally:
  1902. if sources:
  1903. sources.close()
  1904. os.chown(sources_file, self.opts.u, self.opts.g)
  1905. def genLiveMedia(self):
  1906. '''Generate live media iso image.'''
  1907. stage = os.path.join(self.temp, 'staging')
  1908. distro_name = self.conf['distro']['FLL_DISTRO_NAME']
  1909. iso_name = self._getDistroMediaName() + '.iso'
  1910. iso_file = os.path.join(self.opts.o, iso_name)
  1911. md5_file = iso_file + '.md5'
  1912. sha256_file = iso_file + '.sha256'
  1913. cmd = 'xorriso -report_about HINT -as mkisofs -graft-points'
  1914. if self.opts.v:
  1915. cmd += ' -v'
  1916. cmd += ' -pad -l'
  1917. cmd2 = ''
  1918. if os.path.isdir(os.path.join(stage, 'boot/grub')):
  1919. cmd += ' -no-emul-boot -boot-load-size 4 -boot-info-table'
  1920. cmd2 = '%s -l %s -i %s' % (os.path.join(self.opts.s, 'gpthybrid'), self.uuid, iso_file)
  1921. if os.path.isfile(os.path.join(stage, 'boot/grub/i386-pc/grub_eltorito')):
  1922. cmd += ' -b boot/grub/i386-pc/grub_eltorito --grub2-boot-info --grub2-mbr %s' % os.path.join(stage, 'boot/grub/i386-pc/boot_hybrid.img')
  1923. else:
  1924. self.log.critical('failed to find grub El Torito image file')
  1925. raise FllError
  1926. cmd += ' --modification-date=%s' % self.uuid.replace("-","")
  1927. if os.path.isfile(os.path.join(stage, 'efi.img')):
  1928. self.log.info('efi.img found, creating a UEFI ready iso.')
  1929. cmd += ' --efi-boot efi.img -efi-boot-part --efi-boot-image'
  1930. cmd += ' --protective-msdos-label -V %s' % distro_name[:32].upper()
  1931. if os.path.isdir(os.path.join(stage, 'boot')):
  1932. cmd += ' --sort-weight 0 / --sort-weight 1 /boot'
  1933. if os.path.isdir(os.path.join(stage, 'boot/grub')):
  1934. cmd += ' --sort-weight 2 /boot/grub'
  1935. cmd += ' -eltorito-alt-boot'
  1936. cmd += ' -isohybrid-gpt-basdat'
  1937. cmd += ' -x iso_uuid '
  1938. cmd += '-x genisoimage.sort'
  1939. cmd += ' -o %s %s' % (iso_file, stage)
  1940. self.log.info('generating iso image of live media...')
  1941. self.log.info('used cmd: %s' % cmd)
  1942. self._execCmd(cmd.split())
  1943. os.chown(iso_file, self.opts.u, self.opts.g)
  1944. self.log.info('calculating md5sum of live media iso image...')
  1945. md5 = None
  1946. try:
  1947. md5 = open(md5_file, 'w')
  1948. md5.write("%s *%s\n" % (self.__md5sum(iso_file),
  1949. os.path.basename(iso_file)))
  1950. except IOError:
  1951. self.log.exception('failed to write md5sums file')
  1952. raise FllError
  1953. finally:
  1954. if md5:
  1955. md5.close()
  1956. os.chown(md5_file, self.opts.u, self.opts.g)
  1957. if self.opts.k:
  1958. self.log.info('signing md5 hash...')
  1959. self._signFile(md5_file)
  1960. os.chown(md5_file + '.gpg', self.opts.u, self.opts.g)
  1961. self.log.info('calculating sha256sum of live media iso image...')
  1962. sha256 = None
  1963. try:
  1964. sha256 = open(sha256_file, 'w')
  1965. sha256.write("%s *%s\n" % (self.__sha256sum(iso_file),
  1966. os.path.basename(iso_file)))
  1967. except IOError:
  1968. self.log.exception('failed to write sha256sums file')
  1969. raise FllError
  1970. finally:
  1971. if sha256:
  1972. sha256.close()
  1973. os.chown(sha256_file, self.opts.u, self.opts.g)
  1974. if self.opts.k:
  1975. self.log.info('signing sha256 hash...')
  1976. self._signFile(sha256_file)
  1977. os.chown(sha256_file + '.gpg', self.opts.u, self.opts.g)
  1978. self._writeManifests(os.path.splitext(iso_file)[0])
  1979. if not self.opts.B:
  1980. self._writeSources(os.path.splitext(iso_file)[0])
  1981. for f in glob.glob('%s*' % os.path.splitext(iso_file)[0]):
  1982. self.log.info(f)
  1983. if self.opts.x:
  1984. self.log.info('-----------------------')
  1985. self.log.info('Links to current build:')
  1986. for f in glob.glob('%s*' % os.path.splitext(iso_file)[0]):
  1987. arch = ''.join(self.conf['archs'].keys())
  1988. self.log.info(' * %s/%s/%s/%s', self.opts.x,
  1989. self.conf['packages']['profile'], arch, os.path.basename(f))
  1990. def buildChroots(self):
  1991. '''Main loop to call all chroot building functions.'''
  1992. archs = self.conf['archs'].keys()
  1993. codename = self.opts.C
  1994. for arch in archs:
  1995. self._bootStrap(arch)
  1996. self._dpkgAddDivert(arch)
  1997. self._defaultEtc(arch)
  1998. self._distroDefaultEtc(arch)
  1999. self._preseedDebconf(arch)
  2000. self._primeApt(arch)
  2001. self._installPkgs(arch, codename)
  2002. self._postInst(arch)
  2003. self._collectManifest(arch)
  2004. self._finalEtc(arch)
  2005. self._dpkgUnDivert(arch)
  2006. self._makeInitramfs(arch)
  2007. self._cleanChroot(arch)
  2008. self._chrootSquashfs(arch)
  2009. self._stageArch(arch)
  2010. self._nukeChroot(arch)
  2011. def main(self):
  2012. '''Main loop.'''
  2013. self.checkOpts()
  2014. self.parseConf()
  2015. self.parsePkgProfile()
  2016. self.stageBuildArea()
  2017. if self.opts.n:
  2018. sys.exit(0)
  2019. self.buildChroots()
  2020. self.secondSquashfs()
  2021. self.writeBootldr()
  2022. self.writeMd5Sums()
  2023. self.genLiveMedia()
  2024. duration = datetime.datetime.utcnow() - self.time
  2025. self.log.info('build duration was %d minutes and %d seconds' %
  2026. divmod(duration.seconds, 60))
  2027. if __name__ == '__main__':
  2028. p = optparse.OptionParser(usage = 'fll -c <config file> [-b <directory> ' +
  2029. '-o <directory> -l <file>] [-BdDpqv] [-t <aptcacher>]')
  2030. p.add_option('-a', '--arch', dest = 'a', action = 'store',
  2031. type = 'string', metavar = '<arch>',
  2032. help = 'Build architecture, overrides config file.')
  2033. p.add_option('-b', '--build', dest = 'b', action = 'store',
  2034. type = 'string', metavar = '<directory>',
  2035. help = 'Build directory. A large amount of free space ' +
  2036. 'is required.')
  2037. p.add_option('-B', '--binary', dest = 'B', action = 'store_true',
  2038. help = 'Do binary build only. Disable generation of ' +
  2039. 'URI lists. Default: %default')
  2040. p.add_option('-c', '--config', dest = 'c', action = 'store',
  2041. type = 'string', metavar = '<config file>',
  2042. help = 'Configuration file. This option may be used ' +
  2043. 'more than once to process multiple configurations. ' +
  2044. 'A configuration file must be specified.')
  2045. p.add_option('-d', '--debug', dest = 'd', action = 'store_true',
  2046. help = 'Enable debug mode. Extra output will be ' +
  2047. 'to assist in development. Default: %default')
  2048. p.add_option('-D', '--dummy', dest = 'D', action = 'store_true',
  2049. help = 'Enable dummy mode. Download only ' +
  2050. 'and no squashfs. Default: %default')
  2051. p.add_option('-g', '--gid', dest = 'g', action = 'store',
  2052. type = 'int', metavar = '<group id>',
  2053. help = optparse.SUPPRESS_HELP)
  2054. p.add_option('-k', '--hashkey', dest = 'k', action = 'store',
  2055. type = 'string', metavar = '<key id>', help = 'Set key ' +
  2056. 'to sign MD5 and SHA256 hashes of the generated ISOs.')
  2057. p.add_option('-l', '--log', dest = 'l', action = 'store',
  2058. type = 'string', metavar = '<file>',
  2059. help = 'Log debug output to file. Note that when ' +
  2060. 'logging is enabled, output to the console is buffered.')
  2061. p.add_option('-n', '--non-root', dest = 'n', action = 'store_true',
  2062. help = optparse.SUPPRESS_HELP)
  2063. p.add_option('-o', '--output', dest = 'o', action = 'store',
  2064. type = 'string', metavar = '<directory>',
  2065. help = 'Output directory, where the product of this ' +
  2066. 'program will be generated.')
  2067. p.add_option('-p', '--profile', dest = 'p', action = 'store',
  2068. type = 'string', metavar = '<profile>',
  2069. help = 'Package profile, overrides config file.')
  2070. p.add_option('-P', '--preserve', dest = 'P', action = 'store_true',
  2071. help = 'Preserve build directory. Disable automatic ' +
  2072. 'cleanup of the build area at exit.')
  2073. p.add_option('-q', '--quiet', dest = 'q', action = 'store_true',
  2074. help = 'Enable quiet mode. Only high priority messages ' +
  2075. 'will be generated.')
  2076. p.add_option('-s', '--share', dest = 's', action = 'store',
  2077. type = 'string', metavar = '<directory>',
  2078. help = optparse.SUPPRESS_HELP)
  2079. p.add_option('-t', '--aptcacher', dest = 't', action = 'store',
  2080. type = 'string', metavar = '<aptcacher>',
  2081. help = 'apt-cacher uri. Note that all r.cached are new set!')
  2082. p.add_option('-u', '--uid', dest = 'u', action = 'store',
  2083. type = 'int', metavar = '<user id>',
  2084. help = optparse.SUPPRESS_HELP)
  2085. p.add_option('-v', '--verbose', dest = 'v', action = 'store_true',
  2086. help = 'Enable verbose mode. All messages will be ' +
  2087. 'generated, such as announcing current operation.')
  2088. p.add_option('-x', '--builderuri', dest = 'x', action = 'store',
  2089. type = 'string', metavar = '<builder uri>',
  2090. help = 'Builder-Uri like http(s)://domain.tld - Base for ' +
  2091. 'direct download links for build files.')
  2092. p.set_defaults(b = None, B = False, d = False, D = False, g = os.getgid(),
  2093. k = None, l = None, n = False, o = None, p = None, P = False,
  2094. q = False, s = None, t = None, u = os.getuid(), v = False,
  2095. x = None)
  2096. options = p.parse_args()[0]
  2097. try:
  2098. fll = FLLBuilder(options)
  2099. fll.main()
  2100. except KeyboardInterrupt:
  2101. pass
  2102. except FllError:
  2103. sys.exit(1)