FIX overwritting packages in bucket, now readds the package to repo if it is overwritten

Author: Janez Justin

rpm/s3rpm.py

@@ -33,8 +33,16 @@ def lambda_handler(event, context):
                 s3.download_file(os.environ['BUCKET_NAME'], s3_repo_dir+'/repodata/'+f, repo.repodir+'repodata/'+f)
             repo.read()
         print('Creating Metadata files')
-        repo, cache = check_changed_files(repo, s3_repo_dir)
-        #Check if object was removed
+        if event['Records'][0]['eventName'].startswith('ObjectCreated'):
+            print('helpme')
+            repo, cache = check_changed_files(repo, s3_repo_dir, newfile=event['Records'][0]['s3']['object']['key'])
+        else:
+            repo, cache = check_changed_files(repo, s3_repo_dir)
+        #save cache to bucket
+        s3 = boto3.resource('s3')
+        f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir+'/repo_cache')
+        print("Writing file: %s" % (str(f_index_obj)))
+        f_index_obj.put(Body=str(json.dumps(cache)))
 
         repo.save()
 
@@ -43,16 +51,12 @@ def lambda_handler(event, context):
             sign_md_file(repo, s3_repo_dir)
 
         #save files to bucket
-        s3 = boto3.resource('s3')
         for f in files:
             with open(repo.repodir+'repodata/'+f, 'rb') as g:
                 f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir+'/repodata/'+f)
                 print("Writing file: %s" % (str(f_index_obj)))
                 f_index_obj.put(Body=g.read(-1), ACL=get_public())
-        f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir+'/repo_cache')
-        print("Writing file: %s" % (str(f_index_obj)))
-        f_index_obj.put(Body=str(json.dumps(cache)))
-
+                
         #Let us clean up
         shutil.rmtree(repo.repodir)
         if os.path.exists('/tmp/gpgdocs'):
@@ -114,14 +118,35 @@ def get_cache(repo, s3_repo_dir):
         cache = {}
     return cache
 
-def check_changed_files(repo, s3_repo_dir):
+def remove_overwritten_file_from_cache(cache, newfile, s3_repo_dir, repo):
+    """
+    remove pkg from metadata and repo
+    """
+    fname = newfile[len(s3_repo_dir):] # '/filename.rpm' - without path
+    print('file %s has been overwritten and will be removed from md and repo' % (fname))
+    pkg_id = cache[fname]
+    del cache[fname]
+    
+    # save cache in case new event occurs
+    s3 = boto3.resource('s3')
+    f_index_obj = s3.Object(bucket_name=os.environ['BUCKET_NAME'], key=s3_repo_dir+'/repo_cache')
+    f_index_obj.put(Body=str(json.dumps(cache)))
+
+    repo.remove_package(pkg_id)    
+    return cache
+
+
+def check_changed_files(repo, s3_repo_dir,newfile=None):
     """
     check if there are any new files in bucket or any deleted files
     """
     print("Checking for changes : %s" % (s3_repo_dir))
     cache = get_cache(repo, s3_repo_dir)
     s3 = boto3.resource('s3')
     files = []
+    #if file was overwriten and is in repocache then remove it from cache, so next for loop will add back the new
+    if newfile != None and newfile[len(s3_repo_dir):] in cache:
+        cache = remove_overwritten_file_from_cache(cache, newfile, s3_repo_dir, repo)
     #cycle through all objects ending with .rpm in REPO_DIR and check if they are already in repodata, if not add them
     for obj in s3.Bucket(os.environ['BUCKET_NAME']).objects.filter(Prefix=s3_repo_dir):
         files.append(obj.key)

rpm/s3rpm_test.py

@@ -145,7 +145,7 @@ def tearDown(self):
     @patch('s3rpm.get_cache')
     @patch('s3rpm.YumRepository')
     @patch('s3rpm.boto3')
-    def test_defined_repodir(self, s3_mock, yum_mock, cache_mock, ):
+    def test_defined_repodir(self, s3_mock, yum_mock, cache_mock):
         cache_mock.return_value = {"pkgname":"ID"}
 
         yum_mock.return_value = MagicMock(repodir='test_s3rpm/')
@@ -178,6 +178,6 @@ def test_bad_bucket_name(self, s3_mock):
         s3rpm.lambda_handler(S3_EVENT, {})
         s3_mock.client.assert_called_with('s3')
         self.assertEqual(len(s3_mock.resource().Object().put.mock_calls), 0)
-S3_EVENT = {"Records":[{"s3": {"object": {"key": "test_s3rpm/repo/pkgname-0.3.8-x86_64.rpm",},"bucket": {"name": "bucket",},},"eventName": "ObjectCreated:Put"}]}
+S3_EVENT = {"Records":[{"s3": {"object": {"key": "test_s3rpm/repo/pkgname-0.3.8-x86_64.rpm",},"bucket": {"name": "bucket",},},"eventName": "ObjectCreated:*"}]}
 if __name__ == '__main__':
     unittest.main()