import repository from arizona
[raven.git] / 2.0 / python / s3tool.py
1 #! /usr/bin/env python
2 """
3 <Program Name>
4    s3test.py
5
6 <Started>
7    6/5/07
8
9 <Author>
10    Programmed by Scott Baker
11
12 <Purpose>
13    Test for storks3
14 """
15
16 #           [option, long option,    variable,     action,        data,     default,                            metavar, description]
17 """arizonaconfig
18    options=[
19             ["-C",   "--configfile", "configfile", "store",       "string", "/usr/local/stork/etc/stork.conf", "FILE",   "use a different config file (/usr/local/stork/etc/stork.conf is the default)"]]
20    includes=[]
21 """
22
23
24 import arizonaconfig
25 import arizonareport
26 import storks3
27 import os
28 import sys
29 import shutil
30
31 def try_remove(fn):
32       # remove the file if it exists
33       try:
34          os.remove(fn)
35       except OSError:
36          pass
37
38 def areyousure(prompt):
39    print prompt
40    print "Are you sure (y/N) ? ",
41    
42    result = sys.stdin.readline()
43    if (result[0] != 'y') and (result[0] != 'Y'):
44        print "aborted by user request"
45        sys.exit(1)
46
47 def download_repo_Tar():
48    # parse command line and options
49    args = arizonaconfig.init_options("s3downloadrepo.py", configfile_optvar="configfile", version="2.0")
50
51    arizonaconfig.set_option("awsbucket", "stork-repository-import")
52
53    storks3.init()
54
55    print "reading directory"
56    storks3.dumpdir()
57
58    print "getting repository tar files..."
59
60    for entry in storks3.readdir():
61       if not entry.key.startswith("reposplit"):
62           print "   ignoring: " + entry.key
63           continue
64
65       fn = os.path.join("/mnt", entry.key)
66       print "   getting " + entry.key + " to " + fn
67       try:
68           storks3.get_file(entry.key, fn)
69       except TypeError, e:
70           print "download of " + entry.key + " failed with error " + str(e) + " trying again"
71           try:
72               storks3.get_file(entry.key, fn)
73           except TypeError, e:
74               print "upload of " + entry.key + " failed with error " + str(e) + " gave up"
75
76 def upload_repo_tar():
77    arizonaconfig.set_option("awsbucket", "stork-repository-import")
78
79    areyousure("the repository tarball on S3 will be replaced")
80
81    storks3.init()
82
83    print "reading directory"
84    storks3.dumpdir()
85
86    print "putting repository tar files..."
87
88    for entry in os.listdir("/mnt"):
89        if entry.startswith("reposplit"):
90            fn = os.path.join("/mnt", entry)
91            print "  putting " + str(fn)
92            try:
93                storks3.put_file(fn, entry)
94            except TypeError, e:
95                print "upload of " + fn + " failed with error " + str(e) + " trying again"
96                try:
97                    storks3.put_file(fn, entry)
98                except TypeError, e:
99                    print "upload of " + fn + " failed with error " + str(e) + " gave up"
100
101    print "reading directory"
102    storks3.dumpdir()
103
104 def destroy_repo():
105    areyousure("the repository contents on S3 will be destroyed")
106
107    storks3.init()
108    storks3.destroy()
109
110 def dump_repo():
111    storks3.init()
112    storks3.dumpdir()
113
114 def list_buckets():
115    storks3.init()
116    storks3.dumpbuckets()
117
118 def put_file(name):
119    storks3.init()
120    storks3.put_file(name, os.path.basename(name))
121
122 def download_repo():
123    storks3.init()
124
125    entries = storks3.readdir()
126    for entry in entries:
127       tempname = os.path.join("/tmp", entry.key)
128       print "downloading: " + entry.key
129       try:
130          metadata = storks3.get_file(entry.key, tempname)
131       except:
132          arizonareport.send_error(0, "failed to get " + entry.key)
133          continue
134
135       kind = metadata.get("kind", None)
136
137       local_filename = metadata.get("local-filename", None)
138       if not local_filename:
139           arizonareport.send_error(0, "failed to determine local filename for " + entry.key)
140           continue
141
142       # create the directory if it does not exist
143       try:
144          os.makedirs(os.path.dirname(local_filename))
145       except OSError:
146          pass
147
148       try_remove(local_filename)
149
150       try:
151          shutil.move(tempname, local_filename)
152       except OSError:
153          arizonareport.send_error(0, "failed to move file to local_filename: " + local_filename)
154          continue
155
156       if kind == "package":
157          metafile_name = metadata.get("metafile", None)
158          if metafile_name:
159             metalink_name = local_filename + ".metalink"
160
161             # remove the old metalink symlink
162             try_remove(metalink_name)
163
164             try:
165                os.symlink(metafile_name, metalink_name)
166             except OSError:
167                arizonareport.send_error(0, "failed to create metalink " + metalink_name + " to " + metafile_name)
168
169
170
171 def help():
172    arizonareport.send_out(0,"syntax: s3tool <command>")
173    arizonareport.send_out(0,"")
174    arizonareport.send_out(0,"repo tarball commands:")
175    arizonareport.send_out(0,"   downloadrepotar: download the repository tarball to /mnt")
176    arizonareport.send_out(0,"   uploadrepotar: upload the repository tarball to /mnt")
177    arizonareport.send_out(0,"")
178    arizonareport.send_out(0,"repository commands:")
179    arizonareport.send_out(0,"   destroy: destroy the repository")
180    arizonareport.send_out(0,"   dump: list the contents of the repository")
181    arizonareport.send_out(0,"   downloadrepo: download repository")
182    sys.exit(1)
183
184 def main():
185    # parse command line and options
186    args = arizonaconfig.init_options("s3uploadrepo.py", configfile_optvar="configfile", version="2.0")
187
188    if len(args) == 0:
189       help()
190
191    if args[0] == "downloadrepotar":
192       download_repo_tar()
193    elif args[0] == "uploadrepotar":
194       upload_repo_tar()
195    elif args[0] == "destroy":
196       destroy_repo()
197    elif args[0] == "dump":
198       dump_repo()
199    elif args[0] == "listbuckets":
200       list_buckets()
201    elif args[0] == "downloadrepo":
202       download_repo()
203    elif args[0] == "putfile":
204       put_file(args[1])
205    else:
206       help()
207
208 if __name__ == "__main__":
209    main()