| 1 |
#!/usr/bin/python2.6 |
| 2 |
# coding=utf-8 |
| 3 |
# |
| 4 |
# $Id$ |
| 5 |
|
| 6 |
import ConfigParser |
| 7 |
import datetime |
| 8 |
import getpass |
| 9 |
import hashlib |
| 10 |
import itertools |
| 11 |
import logging |
| 12 |
import optparse |
| 13 |
import os |
| 14 |
import os.path |
| 15 |
import progressbar |
| 16 |
import progressbar.widgets |
| 17 |
import re |
| 18 |
import socket |
| 19 |
import sqlobject |
| 20 |
import sys |
| 21 |
|
| 22 |
from sqlobject import sqlbuilder |
| 23 |
from Cheetah.Template import Template |
| 24 |
|
| 25 |
from lib.python import catalog |
| 26 |
from lib.python import checkpkg_lib |
| 27 |
from lib.python import common_constants |
| 28 |
from lib.python import configuration |
| 29 |
from lib.python import database |
| 30 |
from lib.python import models as m |
| 31 |
from lib.python import package_checks |
| 32 |
from lib.python import package_stats |
| 33 |
from lib.python import rest |
| 34 |
from lib.python import shell |
| 35 |
from lib.python import struct_util |
| 36 |
from lib.python import system_pkgmap |
| 37 |
|
| 38 |
USAGE = """ |
| 39 |
Preparing the database: |
| 40 |
%prog initdb |
| 41 |
%prog system-metadata-to-disk [ <infile-contents> <infile-pkginfo> |
| 42 |
[ <outfile> [ <osrel> <arch> ] ] ] |
| 43 |
%prog import-system-metadata <osrel> <arch> |
| 44 |
|
| 45 |
Managing individual packages: |
| 46 |
%prog importpkg <file1> [ ... ] |
| 47 |
%prog removepkg <md5sum> [ ... ] |
| 48 |
|
| 49 |
Managing catalogs: |
| 50 |
%prog add-to-cat <osrel> <arch> <cat-release> <md5sum> [ ... ] |
| 51 |
%prog del-from-cat <osrel> <arch> <cat-release> <md5sum> [ ... ] |
| 52 |
%prog sync-cat-from-file <osrel> <arch> <cat-release> <catalog-file> |
| 53 |
%prog sync-catalogs-from-tree <cat-release> <opencsw-dir> |
| 54 |
%prog show cat [options] |
| 55 |
|
| 56 |
Inspecting individual packages: |
| 57 |
%prog show errors <md5sum> [ ... ] |
| 58 |
%prog show pkg <pkgname> [ ... ] |
| 59 |
%prog gen-html <md5sum> [ ... ] |
| 60 |
%prog pkg search <catalogname> |
| 61 |
%prog show basename [options] <filename> |
| 62 |
%prog show filename [options] <filename> |
| 63 |
%prog show files <md5-sum> |
| 64 |
|
| 65 |
osrel := SunOS5.8 | SunOS5.9 | SunOS5.10 | SunOS5.11 |
| 66 |
arch := i386 | sparc |
| 67 |
|
| 68 |
Examples: |
| 69 |
%prog add-to-cat SunOS5.9 sparc unstable <md5sum> |
| 70 |
%prog del-from-cat SunOS5.10 i386 testing <md5sum> |
| 71 |
""" |
| 72 |
|
| 73 |
SHOW_PKG_TMPL = """catalogname: $catalogname |
| 74 |
pkgname: $pkginst.pkgname |
| 75 |
basename: $basename |
| 76 |
mtime: $mtime |
| 77 |
md5_sum: $md5_sum |
| 78 |
arch: $arch.name |
| 79 |
os_rel: $os_rel.short_name |
| 80 |
maintainer: $maintainer.email |
| 81 |
version_string: $version_string |
| 82 |
rev: $rev |
| 83 |
stats_version: $stats_version |
| 84 |
""" |
| 85 |
|
| 86 |
DEFAULT_TEMPLATE_FILENAME = "../lib/python/pkg-review-template.html" |
| 87 |
CATALOGS_ALLOWED_TO_GENERATE = frozenset([ |
| 88 |
"beanie", |
| 89 |
"bratislava", |
| 90 |
"dublin", |
| 91 |
"kiel", |
| 92 |
"munich", |
| 93 |
"unstable", |
| 94 |
]) |
| 95 |
CATALOGS_ALLOWED_TO_BE_IMPORTED = frozenset([ |
| 96 |
"beanie", |
| 97 |
"bratislava", |
| 98 |
"dublin", |
| 99 |
"kiel", |
| 100 |
"legacy", |
| 101 |
"munich", |
| 102 |
"unstable", |
| 103 |
]) |
| 104 |
|
| 105 |
|
| 106 |
class Error(Exception): |
| 107 |
"Generic error in the program." |
| 108 |
|
| 109 |
|
| 110 |
class UsageError(Error): |
| 111 |
"Error in command line options." |
| 112 |
|
| 113 |
|
| 114 |
class OpencswTreeError(Error): |
| 115 |
"A problem with the OpenCSW directory tree." |
| 116 |
|
| 117 |
|
| 118 |
class HtmlGenerator(object): |
| 119 |
|
| 120 |
def __init__(self, identifiers, template=None, rest_client=None, debug=False): |
| 121 |
"""Initialize the object |
| 122 |
|
| 123 |
args: identifiers: md5 sums or file names of packages |
| 124 |
template: Optional HTML template |
| 125 |
""" |
| 126 |
self.template = template |
| 127 |
self.identifiers = identifiers |
| 128 |
self.debug = debug |
| 129 |
assert rest_client is not None, 'You need to pass rest_client here' |
| 130 |
self.rest_client = rest_client |
| 131 |
|
| 132 |
def GetErrorTagsResult(self, srv4): |
| 133 |
res = m.CheckpkgErrorTag.select( |
| 134 |
m.CheckpkgErrorTag.q.srv4_file==srv4) |
| 135 |
return res |
| 136 |
|
| 137 |
def GetOverrideResult(self, srv4): |
| 138 |
res = m.CheckpkgOverride.select( |
| 139 |
m.CheckpkgOverride.q.srv4_file==srv4) |
| 140 |
return res |
| 141 |
|
| 142 |
def GenerateHtml(self): |
| 143 |
pkgstats = [] |
| 144 |
# Add error tags |
| 145 |
for identifier in self.identifiers: |
| 146 |
srv4 = GetPkg(identifier) |
| 147 |
data = self.rest_client.GetPkgstatsByMd5(srv4.md5_sum) |
| 148 |
build_src_url_svn = srv4.GetSvnUrl() |
| 149 |
build_src_url_trac = srv4.GetTracUrl() |
| 150 |
if "OPENCSW_REPOSITORY" in data["pkginfo"]: |
| 151 |
data["build_src"] = data["pkginfo"]["OPENCSW_REPOSITORY"] |
| 152 |
else: |
| 153 |
data["build_src"] = u"Repository address unknown" |
| 154 |
data["build_src_url_svn"] = build_src_url_svn |
| 155 |
data["build_src_url_trac"] = build_src_url_trac |
| 156 |
data["error_tags"] = list(self.GetErrorTagsResult(srv4)) |
| 157 |
data["overrides"] = list(self.GetOverrideResult(srv4)) |
| 158 |
pkgstats.append(data) |
| 159 |
# This assumes the program is run as "bin/pkgdb", and not "lib/python/pkgdb.py". |
| 160 |
if not self.template: |
| 161 |
tmpl_filename = os.path.join(os.path.split(__file__)[0], |
| 162 |
DEFAULT_TEMPLATE_FILENAME) |
| 163 |
else: |
| 164 |
tmpl_filename = self.template |
| 165 |
tmpl_str = open(tmpl_filename, "r").read() |
| 166 |
t = Template(tmpl_str, searchList=[{ |
| 167 |
"pkgstats": pkgstats, |
| 168 |
"hachoir_machines": common_constants.MACHINE_ID_METADATA, |
| 169 |
}]) |
| 170 |
return unicode(t) |
| 171 |
|
| 172 |
|
| 173 |
def NormalizeId(some_id): |
| 174 |
"""Used to normalize identifiers (md5, filename). |
| 175 |
|
| 176 |
Currently, strips paths off given package paths.""" |
| 177 |
if not struct_util.IsMd5(some_id): |
| 178 |
logging.warning( |
| 179 |
"Given Id (%s) is not an md5 sum. Will attempt to retrieve " |
| 180 |
"it from the datbase, but it might fail.", |
| 181 |
repr(some_id)) |
| 182 |
return os.path.basename(some_id) |
| 183 |
|
| 184 |
|
| 185 |
def GetPkg(some_id): |
| 186 |
some_id = NormalizeId(some_id) |
| 187 |
logging.debug("Selecting from db: %s", repr(some_id)) |
| 188 |
res = m.Srv4FileStats.select( |
| 189 |
sqlobject.OR( |
| 190 |
m.Srv4FileStats.q.md5_sum==some_id, |
| 191 |
m.Srv4FileStats.q.catalogname==some_id)) |
| 192 |
try: |
| 193 |
srv4 = res.getOne() |
| 194 |
except sqlobject.main.SQLObjectIntegrityError, e: |
| 195 |
logging.warning(e) |
| 196 |
for row in res: |
| 197 |
print "- %s %s %s" % (row.md5_sum, row.version_string, row.mtime) |
| 198 |
raise |
| 199 |
except sqlobject.main.SQLObjectNotFound, e: |
| 200 |
logging.fatal("Could not locate a package identified by %s", |
| 201 |
repr(some_id)) |
| 202 |
raise |
| 203 |
logging.debug("Got: %s", srv4) |
| 204 |
return srv4 |
| 205 |
|
| 206 |
|
| 207 |
class CatalogImporter(object): |
| 208 |
|
| 209 |
def __init__(self, debug=False): |
| 210 |
self.debug = debug |
| 211 |
config = configuration.GetConfig() |
| 212 |
username, password = rest.GetUsernameAndPassword() |
| 213 |
self.rest_client = rest.RestClient( |
| 214 |
pkgdb_url=config.get('rest', 'pkgdb'), |
| 215 |
releases_url=config.get('rest', 'releases'), |
| 216 |
username=username, |
| 217 |
password=password, |
| 218 |
debug=debug) |
| 219 |
|
| 220 |
def SyncFromCatalogFile(self, osrel, arch, catrel, catalog_file, |
| 221 |
force_unpack=False): |
| 222 |
"""Syncs a given catalog from a catalog file. |
| 223 |
|
| 224 |
Imports srv4 files if necessary. |
| 225 |
""" |
| 226 |
if catrel not in CATALOGS_ALLOWED_TO_BE_IMPORTED: |
| 227 |
raise UsageError("Catalogs that can be imported: %s" |
| 228 |
% CATALOGS_ALLOWED_TO_BE_IMPORTED) |
| 229 |
|
| 230 |
catalog_dir = os.path.dirname(catalog_file) |
| 231 |
# The plan: |
| 232 |
# - read in the catalog file, and build a md5-filename correspondence |
| 233 |
# data structure. |
| 234 |
logging.debug("Reading the catalog file from disk.") |
| 235 |
src_catalog = catalog.OpencswCatalog(open(catalog_file, "rb")) |
| 236 |
catalog_data = src_catalog.GetCatalogData() |
| 237 |
cat_entry_by_md5 = {} |
| 238 |
cat_entry_by_basename = {} |
| 239 |
for catalog_entry in catalog_data: |
| 240 |
cat_entry_by_md5[catalog_entry["md5sum"]] = catalog_entry |
| 241 |
cat_entry_by_basename[catalog_entry["file_basename"]] = catalog_entry |
| 242 |
|
| 243 |
# - import all srv4 files that were not in the database so far |
| 244 |
entries_to_import = [] |
| 245 |
|
| 246 |
logging.debug("Checking which srv4 files have already been indexed.") |
| 247 |
existence_data = ( |
| 248 |
self.rest_client.BulkQueryStatsExistence(list(cat_entry_by_md5))) |
| 249 |
entries_to_import = [cat_entry_by_md5[x] |
| 250 |
for x in existence_data['missing_stats']] |
| 251 |
|
| 252 |
md5_sums = [] |
| 253 |
if entries_to_import: |
| 254 |
collector = package_stats.StatsCollector(logger=logging, debug=self.debug) |
| 255 |
for entry in entries_to_import: |
| 256 |
entry['pkg_path'] = os.path.join(catalog_dir, entry['file_basename']) |
| 257 |
if len(entries_to_import) < 15: |
| 258 |
logging.info("Srv4 files to unpack:") |
| 259 |
for basename in sorted(x['file_basename'] for x in entries_to_import): |
| 260 |
logging.info(" + %s", basename) |
| 261 |
else: |
| 262 |
logging.info('Importing %d packages.', len(entries_to_import)) |
| 263 |
md5_sums = collector.CollectStatsFromCatalogEntries( |
| 264 |
entries_to_import, force_unpack=force_unpack) |
| 265 |
|
| 266 |
# - sync the specific catalog |
| 267 |
# - find the md5 sum list of the current catalog |
| 268 |
logging.debug("Retrieving current catalog assigments from the db.") |
| 269 |
sqo_osrel = m.OsRelease.selectBy(short_name=osrel).getOne() |
| 270 |
sqo_arch = m.Architecture.selectBy(name=arch).getOne() |
| 271 |
sqo_catrel = m.CatalogRelease.selectBy(name=catrel).getOne() |
| 272 |
res = m.Srv4FileInCatalog.select( |
| 273 |
sqlobject.AND( |
| 274 |
m.Srv4FileInCatalog.q.osrel==sqo_osrel, |
| 275 |
m.Srv4FileInCatalog.q.arch==sqo_arch, |
| 276 |
m.Srv4FileInCatalog.q.catrel==sqo_catrel)) |
| 277 |
db_srv4s_in_cat_by_md5 = {} |
| 278 |
# TODO(maciej): Convert package removals to REST. Maybe in bulk? |
| 279 |
for srv4_in_cat in res: |
| 280 |
try: |
| 281 |
srv4 = srv4_in_cat.srv4file |
| 282 |
if srv4.use_to_generate_catalogs: |
| 283 |
db_srv4s_in_cat_by_md5[srv4.md5_sum] = srv4_in_cat |
| 284 |
except sqlobject.main.SQLObjectNotFound as e: |
| 285 |
logging.warning("Could not retrieve a srv4 file from the db: %s", e) |
| 286 |
# Since the srv4_in_cat object has lost its reference, there's no use |
| 287 |
# keeping it around. |
| 288 |
srv4_in_cat.destroySelf() |
| 289 |
|
| 290 |
disk_md5s = set(cat_entry_by_md5) |
| 291 |
db_md5s = set(db_srv4s_in_cat_by_md5) |
| 292 |
# - match the md5 sum lists between db and disk |
| 293 |
md5_sums_to_add = disk_md5s.difference(db_md5s) |
| 294 |
md5_sums_to_remove = db_md5s.difference(disk_md5s) |
| 295 |
logging.info("There are %s packages to remove and %s to add", |
| 296 |
len(md5_sums_to_remove), |
| 297 |
len(md5_sums_to_add)) |
| 298 |
if md5_sums_to_remove: |
| 299 |
logging.info("To remove from %s %s %s:", osrel, arch, catrel) |
| 300 |
for md5 in md5_sums_to_remove: |
| 301 |
logging.info( |
| 302 |
" - %s", |
| 303 |
db_srv4s_in_cat_by_md5[md5].srv4file.basename) |
| 304 |
|
| 305 |
if md5_sums_to_add and len(md5_sums_to_add) < 15: |
| 306 |
logging.info("To add to from %s %s %s:", osrel, arch, catrel) |
| 307 |
for md5 in md5_sums_to_add: |
| 308 |
logging.info( |
| 309 |
" + %s", |
| 310 |
cat_entry_by_md5[md5]["file_basename"]) |
| 311 |
|
| 312 |
user = getpass.getuser() |
| 313 |
# Remove |
| 314 |
# We could use checkpkg_lib.Catalog.RemoveSrv4(), but it would redo |
| 315 |
# many of the database queries and would be much slower. |
| 316 |
if md5_sums_to_remove: |
| 317 |
logging.info("Removing %d packages from the %s %s %s catalog.", |
| 318 |
len(md5_sums_to_remove), osrel, arch, catrel) |
| 319 |
for md5 in md5_sums_to_remove: |
| 320 |
db_srv4s_in_cat_by_md5[md5].destroySelf() |
| 321 |
|
| 322 |
if not md5_sums_to_add: |
| 323 |
return |
| 324 |
logging.info("Adding %d packages to the %s %s %s catalog.", |
| 325 |
len(md5_sums_to_add), |
| 326 |
osrel, arch, catrel) |
| 327 |
pbar = progressbar.ProgressBar(widgets=[ |
| 328 |
progressbar.widgets.Percentage(), |
| 329 |
' ', |
| 330 |
progressbar.widgets.ETA(), |
| 331 |
' ', |
| 332 |
progressbar.widgets.Bar() |
| 333 |
]) |
| 334 |
pbar.maxval = len(md5_sums_to_add) |
| 335 |
pbar.start() |
| 336 |
counter = itertools.count(1) |
| 337 |
for md5 in md5_sums_to_add: |
| 338 |
logging.debug("Adding %s", cat_entry_by_md5[md5]["file_basename"]) |
| 339 |
|
| 340 |
# Explicitly calling the RegisterLevelTwo function, in case the |
| 341 |
# packages have the flag "use_in_catalogs" set to False. Calling |
| 342 |
# this function will set it to True. |
| 343 |
# Note: This makes populating catalogs really, really slow, and |
| 344 |
# causes subsequent runs to be slow as well. It should only be |
| 345 |
# a temporary measure. |
| 346 |
try: |
| 347 |
pkg = m.Srv4FileStats.selectBy(md5_sum=md5).getOne() |
| 348 |
if (not pkg.registered_level_two or |
| 349 |
not pkg.use_to_generate_catalogs): |
| 350 |
self.rest_client.RegisterLevelTwo(md5, use_in_catalogs=True) |
| 351 |
except sqlobject.main.SQLObjectNotFound: |
| 352 |
pass |
| 353 |
|
| 354 |
# No need to explicitly register the package here; the REST |
| 355 |
# interface will implicitly take care of that (except it will not |
| 356 |
# touch the "use_package_in_catalogs" flag. |
| 357 |
self.rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5) |
| 358 |
pbar.update(counter.next()) |
| 359 |
pbar.finish() |
| 360 |
|
| 361 |
def SyncFromCatalogTree(self, catrel, base_dir, force_unpack=False): |
| 362 |
logging.debug("SyncFromCatalogTree(%s, %s, force_unpack=%s)", |
| 363 |
repr(catrel), repr(base_dir), force_unpack) |
| 364 |
if not os.path.isdir(base_dir): |
| 365 |
raise UsageError("%s is not a diractory" % repr(base_dir)) |
| 366 |
if catrel not in common_constants.DEFAULT_CATALOG_RELEASES: |
| 367 |
logging.warning( |
| 368 |
"The catalog release %s is not one of the default releases.", |
| 369 |
repr(catrel)) |
| 370 |
sqo_catrel = m.CatalogRelease.selectBy(name=catrel).getOne() |
| 371 |
for osrel in common_constants.OS_RELS: |
| 372 |
logging.info(" OS release: %s", repr(osrel)) |
| 373 |
sqo_osrel = m.OsRelease.selectBy(short_name=osrel).getOne() |
| 374 |
for arch in common_constants.PHYSICAL_ARCHITECTURES: |
| 375 |
logging.info(" Architecture: %s", repr(arch)) |
| 376 |
sqo_arch = m.Architecture.selectBy(name=arch).getOne() |
| 377 |
catalog_file = self.ComposeCatalogFilePath(base_dir, osrel, arch) |
| 378 |
if not os.path.exists(catalog_file): |
| 379 |
logging.warning("Could not find %s, skipping.", repr(catalog_file)) |
| 380 |
continue |
| 381 |
logging.info(" %s", catalog_file) |
| 382 |
self.SyncFromCatalogFile(osrel, arch, catrel, catalog_file, |
| 383 |
force_unpack=force_unpack) |
| 384 |
|
| 385 |
def ComposeCatalogFilePath(self, base_dir, osrel, arch): |
| 386 |
short_osrel = osrel.replace("SunOS", "") |
| 387 |
return os.path.join(base_dir, arch, short_osrel, "catalog") |
| 388 |
|
| 389 |
|
| 390 |
def main(): |
| 391 |
parser = optparse.OptionParser(USAGE) |
| 392 |
parser.add_option("-d", "--debug", dest="debug", |
| 393 |
default=False, action="store_true", |
| 394 |
help="Turn on debugging messages") |
| 395 |
parser.add_option("-t", "--pkg-review-template", dest="pkg_review_template", |
| 396 |
help="A Cheetah template used for package review reports.") |
| 397 |
parser.add_option("-r", "--os-release", dest="osrel", |
| 398 |
default="SunOS5.10", |
| 399 |
help="E.g. SunOS5.10") |
| 400 |
parser.add_option("-a", "--arch", dest="arch", |
| 401 |
default="sparc", |
| 402 |
help="'i386' or 'sparc'") |
| 403 |
parser.add_option("-c", "--catalog-release", dest="catrel", |
| 404 |
default="unstable", |
| 405 |
help="E.g. unstable, dublin") |
| 406 |
parser.add_option("--replace", dest="replace", |
| 407 |
default=False, action="store_true", |
| 408 |
help="Replace packages when importing (importpkg)") |
| 409 |
parser.add_option("--profile", dest="profile", |
| 410 |
default=False, action="store_true", |
| 411 |
help="Turn on profiling") |
| 412 |
parser.add_option("--force-unpack", dest="force_unpack", |
| 413 |
default=False, action="store_true", |
| 414 |
help="Force unpacking of packages") |
| 415 |
options, args = parser.parse_args() |
| 416 |
|
| 417 |
logging_level = logging.INFO |
| 418 |
if options.debug: |
| 419 |
logging_level = logging.DEBUG |
| 420 |
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' |
| 421 |
logging.basicConfig(format=fmt, level=logging_level) |
| 422 |
|
| 423 |
if not args: |
| 424 |
raise UsageError("Please specify a command. See --help.") |
| 425 |
# SetUpSqlobjectConnection needs to be called after |
| 426 |
# logging.basicConfig |
| 427 |
configuration.SetUpSqlobjectConnection() |
| 428 |
command = args[0] |
| 429 |
args = args[1:] |
| 430 |
if command == 'show': |
| 431 |
subcommand = args[0] |
| 432 |
args = args[1:] |
| 433 |
elif command == 'pkg': |
| 434 |
subcommand = args[0] |
| 435 |
args = args[1:] |
| 436 |
else: |
| 437 |
subcommand = None |
| 438 |
|
| 439 |
md5_sums = args |
| 440 |
|
| 441 |
if (command, subcommand) == ('show', 'errors'): |
| 442 |
for md5_sum in md5_sums: |
| 443 |
srv4 = GetPkg(md5_sum) |
| 444 |
res = m.CheckpkgErrorTag.select(m.CheckpkgErrorTag.q.srv4_file==srv4) |
| 445 |
for row in res: |
| 446 |
print 'overridden' if row.overridden else 'active', |
| 447 |
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name, |
| 448 |
print row.os_rel.short_name |
| 449 |
elif (command, subcommand) == ('show', 'overrides'): |
| 450 |
for md5_sum in md5_sums: |
| 451 |
srv4 = GetPkg(md5_sum) |
| 452 |
res = m.CheckpkgOverride.select(m.CheckpkgOverride.q.srv4_file==srv4) |
| 453 |
for row in res: |
| 454 |
print row.pkgname, row.tag_name, row.tag_info |
| 455 |
elif (command, subcommand) == ('show', 'pkg'): |
| 456 |
for md5_sum in md5_sums: |
| 457 |
srv4 = GetPkg(md5_sum) |
| 458 |
t = Template(SHOW_PKG_TMPL, searchList=[srv4]) |
| 459 |
sys.stdout.write(unicode(t)) |
| 460 |
elif command == 'gen-html': |
| 461 |
config = configuration.GetConfig() |
| 462 |
username, password = rest.GetUsernameAndPassword() |
| 463 |
rest_client = rest.RestClient( |
| 464 |
pkgdb_url=config.get('rest', 'pkgdb'), |
| 465 |
releases_url=config.get('rest', 'releases'), |
| 466 |
username=username, |
| 467 |
password=password, |
| 468 |
debug=options.debug) |
| 469 |
|
| 470 |
g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client, options.debug) |
| 471 |
sys.stdout.write(g.GenerateHtml()) |
| 472 |
elif command == 'initdb': |
| 473 |
config = configuration.GetConfig() |
| 474 |
database.InitDB(config) |
| 475 |
elif command == 'importpkg': |
| 476 |
collector = package_stats.StatsCollector( |
| 477 |
logger=logging, |
| 478 |
debug=options.debug) |
| 479 |
file_list = args |
| 480 |
catalog_entries = [] |
| 481 |
for file_name in file_list: |
| 482 |
file_hash = hashlib.md5() |
| 483 |
chunk_size = 2 * 1024 * 1024 |
| 484 |
with open(file_name, 'rb') as fd: |
| 485 |
data = fd.read(chunk_size) |
| 486 |
while data: |
| 487 |
file_hash.update(data) |
| 488 |
data = fd.read(chunk_size) |
| 489 |
data_md5_sum = file_hash.hexdigest() |
| 490 |
catalog_entry = { |
| 491 |
'md5sum': data_md5_sum, |
| 492 |
'file_basename': os.path.basename(file_name), |
| 493 |
'pkg_path': file_name, |
| 494 |
} |
| 495 |
catalog_entries.append(catalog_entry) |
| 496 |
md5_list = collector.CollectStatsFromCatalogEntries(catalog_entries, |
| 497 |
force_unpack=options.force_unpack) |
| 498 |
config = configuration.GetConfig() |
| 499 |
rest_client = rest.RestClient( |
| 500 |
pkgdb_url=config.get('rest', 'pkgdb'), |
| 501 |
releases_url=config.get('rest', 'releases'), |
| 502 |
debug=options.debug) |
| 503 |
|
| 504 |
for md5_sum in md5_list: |
| 505 |
logging.debug("Importing %s", md5_sum) |
| 506 |
rest_client.RegisterLevelTwo(md5_sum) |
| 507 |
|
| 508 |
elif command == 'removepkg': |
| 509 |
for md5_sum in md5_sums: |
| 510 |
srv4 = GetPkg(md5_sum) |
| 511 |
in_catalogs = list(srv4.in_catalogs) |
| 512 |
if in_catalogs: |
| 513 |
for in_catalog in in_catalogs: |
| 514 |
logging.warning("%s", in_catalog) |
| 515 |
logging.warning( |
| 516 |
"Not removing from the database, because the package " |
| 517 |
"in question is part of at least one catalog.") |
| 518 |
else: |
| 519 |
logging.info("Removing %s", srv4) |
| 520 |
srv4.DeleteAllDependentObjects() |
| 521 |
srv4.destroySelf() |
| 522 |
elif command == 'add-to-cat': |
| 523 |
if len(args) < 4: |
| 524 |
raise UsageError("Not enough arguments, see usage.") |
| 525 |
user = getpass.getuser() |
| 526 |
osrel, arch, catrel = args[:3] |
| 527 |
username, password = rest.GetUsernameAndPassword() |
| 528 |
rest_client = rest.RestClient(username=username, password=password) |
| 529 |
md5_sums = args[3:] |
| 530 |
for md5_sum in md5_sums: |
| 531 |
rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum) |
| 532 |
elif command == 'del-from-cat': |
| 533 |
if len(args) < 4: |
| 534 |
raise UsageError("Not enough arguments, see usage.") |
| 535 |
osrel, arch, catrel= args[:3] |
| 536 |
md5_sums = args[3:] |
| 537 |
username, password = rest.GetUsernameAndPassword() |
| 538 |
rest_client = rest.RestClient(username=username, password=password) |
| 539 |
for md5_sum in md5_sums: |
| 540 |
rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum) |
| 541 |
elif command == 'system-metadata-to-disk': |
| 542 |
logging.debug("Args: %s", args) |
| 543 |
outfile = None |
| 544 |
infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE |
| 545 |
infile_pkginfo = None |
| 546 |
osrel, arch = (None, None) |
| 547 |
if len(args) >= 2: |
| 548 |
infile_contents = args[0] |
| 549 |
infile_pkginfo = args[1] |
| 550 |
if len(args) >= 3: |
| 551 |
outfile = args[2] |
| 552 |
if len(args) >= 4: |
| 553 |
if len(args) == 5: |
| 554 |
osrel, arch = args[3:5] |
| 555 |
else: |
| 556 |
raise UsageError("Wrong number of arguments (%s), see usage." |
| 557 |
% len(args)) |
| 558 |
spi = system_pkgmap.Indexer(outfile, |
| 559 |
infile_contents, |
| 560 |
infile_pkginfo, |
| 561 |
osrel, |
| 562 |
arch) |
| 563 |
spi.IndexAndSave() |
| 564 |
elif command == 'import-system-metadata': |
| 565 |
if len(args) < 2: |
| 566 |
raise UsageError("Usage: ... import-system-metadata <osrel> <arch>") |
| 567 |
osrel = args[0] |
| 568 |
arch = args[1] |
| 569 |
importer = system_pkgmap.InstallContentsImporter(osrel, arch, |
| 570 |
debug=options.debug) |
| 571 |
importer.Import(show_progress=(not options.debug)) |
| 572 |
elif (command, subcommand) == ('pkg', 'search'): |
| 573 |
logging.debug("Searching for %s", args) |
| 574 |
sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne() |
| 575 |
sqo_arch = m.Architecture.selectBy(name=options.arch).getOne() |
| 576 |
sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne() |
| 577 |
if len(args) < 1: |
| 578 |
logging.fatal("Wrong number of arguments: %s", len(args)) |
| 579 |
raise SystemExit |
| 580 |
for catalogname in args: |
| 581 |
join = [ |
| 582 |
sqlbuilder.INNERJOINOn(None, |
| 583 |
m.Srv4FileInCatalog, |
| 584 |
m.Srv4FileInCatalog.q.srv4file==m.Srv4FileStats.q.id), |
| 585 |
] |
| 586 |
res = m.Srv4FileStats.select( |
| 587 |
sqlobject.AND( |
| 588 |
m.Srv4FileInCatalog.q.osrel==sqo_osrel, |
| 589 |
m.Srv4FileInCatalog.q.arch==sqo_arch, |
| 590 |
m.Srv4FileInCatalog.q.catrel==sqo_catrel, |
| 591 |
m.Srv4FileStats.q.catalogname.contains(catalogname), |
| 592 |
m.Srv4FileStats.q.use_to_generate_catalogs==True), |
| 593 |
join=join, |
| 594 |
).orderBy("catalogname") |
| 595 |
for sqo_srv4 in res: |
| 596 |
print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum) |
| 597 |
elif command == 'sync-cat-from-file': |
| 598 |
if len(args) != 4: |
| 599 |
raise UsageError("Wrong number of arguments, see usage.") |
| 600 |
osrel, arch, catrel, catalog_file = args |
| 601 |
ci = CatalogImporter(debug=options.debug) |
| 602 |
ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file) |
| 603 |
elif command == 'sync-catalogs-from-tree': |
| 604 |
if len(args) != 2: |
| 605 |
raise UsageError("Wrong number of arguments, see usage.") |
| 606 |
ci = CatalogImporter(debug=options.debug) |
| 607 |
catrel, base_dir = args |
| 608 |
ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack) |
| 609 |
elif (command, subcommand) == ('show', 'cat'): |
| 610 |
sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad( |
| 611 |
options.osrel, options.arch, options.catrel) |
| 612 |
res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel) |
| 613 |
for obj in res: |
| 614 |
print obj.catalogname, obj.basename, obj.md5_sum |
| 615 |
elif (command, subcommand) == ('show', 'files'): |
| 616 |
md5_sum = args[0] |
| 617 |
join = [ |
| 618 |
sqlbuilder.INNERJOINOn(None, |
| 619 |
m.Srv4FileStats, |
| 620 |
m.CswFile.q.srv4_file==m.Srv4FileStats.q.id), |
| 621 |
] |
| 622 |
res = m.CswFile.select( |
| 623 |
m.Srv4FileStats.q.md5_sum==md5_sum, |
| 624 |
join=join, |
| 625 |
) |
| 626 |
for obj in res: |
| 627 |
print os.path.join(obj.path, obj.basename) |
| 628 |
elif (command, subcommand) == ('show', 'basename'): |
| 629 |
db_catalog = checkpkg_lib.Catalog() |
| 630 |
for arg in args: |
| 631 |
pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename( |
| 632 |
arg, options.osrel, options.arch, options.catrel) |
| 633 |
for file_path in pkgs_by_path: |
| 634 |
print os.path.join(file_path, arg), ", ".join(pkgs_by_path[file_path]) |
| 635 |
elif (command, subcommand) == ('show', 'filename'): |
| 636 |
db_catalog = checkpkg_lib.Catalog() |
| 637 |
for arg in args: |
| 638 |
pkgs = db_catalog.GetPkgByPath( |
| 639 |
arg, options.osrel, options.arch, options.catrel) |
| 640 |
print " ".join(pkgs) |
| 641 |
else: |
| 642 |
raise UsageError("Command unrecognized: %s" % command) |
| 643 |
|
| 644 |
|
| 645 |
if __name__ == '__main__': |
| 646 |
if "--profile" in sys.argv: |
| 647 |
import cProfile |
| 648 |
t_str = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") |
| 649 |
home = os.environ["HOME"] |
| 650 |
cprof_file_name = os.path.join( |
| 651 |
home, ".checkpkg", "run-modules-%s.cprof" % t_str) |
| 652 |
cProfile.run("main()", sort=1, filename=cprof_file_name) |
| 653 |
else: |
| 654 |
main() |