Hopsfs-255: pydoop with golang and java client

pydoop/hadoop_utils.py

@@ -80,56 +80,51 @@ def reset(self):
 
     # note that this can be None even after trying detection
     def hadoop_home(self):
-        #if not self.__hadoop_home:
-        #    hh = os.getenv("HADOOP_HOME", os.getenv("HADOOP_PREFIX"))
-        #    if not hh:
-        #        exe = subprocess.check_output(
-        #            "command -v hadoop", shell=True, universal_newlines=True
-        #        ).strip()
-        #        candidate, child = os.path.split(os.path.dirname(exe))
-        #        if child == "bin" and os.path.isdir(candidate):
-        #            hh = os.environ["HADOOP_HOME"] = candidate
-        #    self.__hadoop_home = hh
-        #return self.__hadoop_home
-        return "/tmp" 
+        if not self.__hadoop_home:
+            hh = os.getenv("HADOOP_HOME", os.getenv("HADOOP_PREFIX"))
+            if not hh:
+                exe = subprocess.check_output(
+                    "command -v hadoop", shell=True, universal_newlines=True
+                ).strip()
+                candidate, child = os.path.split(os.path.dirname(exe))
+                if child == "bin" and os.path.isdir(candidate):
+                    hh = os.environ["HADOOP_HOME"] = candidate
+            self.__hadoop_home = hh
+        return self.__hadoop_home
 
     def hadoop_conf(self):
-        #if not self.__hadoop_conf:
-        #    error = "Hadoop config not found, try setting HADOOP_CONF_DIR"
-        #    try:
-        #        self.__hadoop_conf = os.environ["HADOOP_CONF_DIR"]
-        #    except KeyError:
-        #        hh = self.hadoop_home()
-        #        if not hh:
-        #            raise RuntimeError(error)
-        #        candidate = os.path.join(hh, 'etc', 'hadoop')
-        #        if not os.path.isdir(candidate):
-        #            raise RuntimeError(error)
-        #        self.__hadoop_conf = os.environ["HADOOP_CONF_DIR"] = candidate
-        #return self.__hadoop_conf
-        return "/tmp" 
+        if not self.__hadoop_conf:
+            error = "Hadoop config not found, try setting HADOOP_CONF_DIR"
+            try:
+                self.__hadoop_conf = os.environ["HADOOP_CONF_DIR"]
+            except KeyError:
+                hh = self.hadoop_home()
+                if not hh:
+                    raise RuntimeError(error)
+                candidate = os.path.join(hh, 'etc', 'hadoop')
+                if not os.path.isdir(candidate):
+                    raise RuntimeError(error)
+                self.__hadoop_conf = os.environ["HADOOP_CONF_DIR"] = candidate
+        return self.__hadoop_conf
 
     def hadoop_params(self):
-        #if not self.__hadoop_params:
-        #    params = {}
-        #    hadoop_conf = self.hadoop_conf()
-        #    for n in "hadoop", "core", "hdfs", "mapred":
-        #        fn = os.path.join(hadoop_conf, "%s-site.xml" % n)
-        #        try:
-        #            params.update(parse_hadoop_conf_file(fn))
-        #        except (IOError, HadoopXMLError):
-        #            pass  # silently ignore, as in Hadoop
-        #    self.__hadoop_params = params
-        #return self.__hadoop_params
-        return {}
+        if not self.__hadoop_params:
+            params = {}
+            hadoop_conf = self.hadoop_conf()
+            for n in "hadoop", "core", "hdfs", "mapred":
+                fn = os.path.join(hadoop_conf, "%s-site.xml" % n)
+                try:
+                    params.update(parse_hadoop_conf_file(fn))
+                except (IOError, HadoopXMLError):
+                    pass  # silently ignore, as in Hadoop
+            self.__hadoop_params = params
+        return self.__hadoop_params
 
     def hadoop_classpath(self):
         if not self.__hadoop_classpath:
-            #cp = subprocess.check_output(
-            #    "hadoop classpath --glob", shell=True, universal_newlines=True
-            #).strip()
-            # libhdfs.go does not need these jar files
-            cp = ""
+            cp = subprocess.check_output(
+                "hadoop classpath --glob", shell=True, universal_newlines=True
+            ).strip()
             # older hadoop versions ignore --glob
             if 'hadoop-common' not in cp:
                 cp = ':'.join(':'.join(glob.iglob(_)) for _ in cp.split(':'))

pydoop/hdfs/core/__init__.py

@@ -24,8 +24,8 @@
 
 
 def init():
-    #import pydoop.utils.jvm as jvm
-    #jvm.load_jvm_lib()
+    import pydoop.utils.jvm as jvm
+    jvm.load_jvm_lib()
     try:
         # NOTE: JVM must be already instantiated
         import pydoop.native_core_hdfs

setup.py

@@ -317,7 +317,7 @@ def run(self):
         build.run(self)
         try:
             self.create_tmp()
-            #self.build_java()
+            self.build_java()
         finally:
             # On NFS, if we clean up right away we have issues with
             # NFS handles being still in the directory trees to be