def execute_antlr_test(self, expected_package, target_workdir_fun=None):
    target = self.get_antlr_target()
    context = self.create_context()
    task = self.prepare_execute(context)
    target_workdir_fun = target_workdir_fun or (lambda x: safe_mkdtemp(dir=x))
    target_workdir = target_workdir_fun(task.workdir)

    # Generate code, then create a synthetic target.
    task.execute_codegen(target, target_workdir)
    syn_target = task._inject_synthetic_target(target, target_workdir)

    actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)]
    expected_sources = syn_target.sources_relative_to_source_root()
    self.assertEquals(set(expected_sources), set(actual_sources))

    # and that the synthetic target has a valid source root and the generated sources have the
    # expected java package
    def get_package(path):
      with open(path) as fp:
        for line in fp:
          match = self.PACKAGE_RE.match(line)
          if match:
            return match.group('package_name')
        return None

    for source in syn_target.sources_relative_to_source_root():
      source_path = os.path.join(target_workdir, source)
      self.assertTrue(os.path.isfile(source_path),
                      "{0} is not the source root for {1}".format(target_workdir, source))
      self.assertEqual(expected_package, get_package(source_path))

      self.assertIn(syn_target, context.targets())

    return syn_target

  def execute_antlr4_test(self, expected_package):
    context = self.create_context()
    task = self.execute(context)

    # get the synthetic target from the private graph
    task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java')
    syn_sourceroot = os.path.join(task_outdir, self.PARTS['srcroot'])
    syn_target_name = ('{srcroot}/{dir}.{name}'.format(**self.PARTS)).replace('/', '.')
    syn_address = SyntheticAddress(spec_path=os.path.relpath(syn_sourceroot, self.build_root),
                                   target_name=syn_target_name)
    syn_target = context.build_graph.get_target(syn_address)

    # verify that the synthetic target's list of sources match what are actually created
    def re_relativize(p):
      """Take a path relative to task_outdir, and make it relative to the build_root"""
      return os.path.relpath(os.path.join(task_outdir, p), self.build_root)

    actual_sources = [re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir)]
    self.assertEquals(set(syn_target.sources_relative_to_buildroot()), set(actual_sources))

    # and that the synthetic target has a valid source root and the generated sources have the
    # expected java package
    def get_package(path):
      with open(path) as fp:
        for line in fp:
          match = self.PACKAGE_RE.match(line)
          if match:
            return match.group('package_name')
        return None

    for source in syn_target.sources_relative_to_source_root():
      source_path = os.path.join(syn_sourceroot, source)
      self.assertTrue(os.path.isfile(source_path),
                      "{0} is not the source root for {1}".format(syn_sourceroot, source))
      self.assertEqual(expected_package, get_package(source_path))

  def execute_antlr_test(self, expected_package):
    context = self.create_context()
    task = self.execute(context)

    # get the synthetic target from the private graph
    task_outdir = os.path.join(task.workdir, 'isolated', self.get_antlr_target().id)

    # verify that the synthetic target's list of sources match what are actually created
    def re_relativize(p):
      """Take a path relative to task_outdir, and make it relative to the build_root"""
      return os.path.relpath(os.path.join(task_outdir, p), self.build_root)

    actual_sources = [re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir)]
    expected_sources = self.get_antlr_syn_target(task).sources_relative_to_buildroot()
    self.assertEquals(set(expected_sources), set(actual_sources))

    # and that the synthetic target has a valid source root and the generated sources have the
    # expected java package
    def get_package(path):
      with open(path) as fp:
        for line in fp:
          match = self.PACKAGE_RE.match(line)
          if match:
            return match.group('package_name')
        return None

    syn_target = self.get_antlr_syn_target(task)
    for source in syn_target.sources_relative_to_source_root():
      source_path = os.path.join(task_outdir, source)
      self.assertTrue(os.path.isfile(source_path),
                      "{0} is not the source root for {1}".format(task_outdir, source))
      self.assertEqual(expected_package, get_package(source_path))

      self.assertIn(syn_target, context.targets())

def path_iterator(args, options):
  for path in args:
    if os.path.isdir(path):
      for filename in Fileset.rglobs('*.py', root=path)():
        yield os.path.join(path, filename), None
    elif os.path.isfile(path):
      yield path, None

  def execute_antlr_test(self, expected_package, target_workdir_fun=None):
    target = self.get_antlr_target()
    context = self.create_context()
    task = self.prepare_execute(context)
    target_workdir_fun = target_workdir_fun or (lambda x: safe_mkdtemp(dir=x))
    # Do not use task.workdir here, because when we calculating hash for synthetic target
    # we need persistent source paths in terms of relative position to build root.
    target_workdir = target_workdir_fun(self.build_root)
    vt = DummyVersionedTarget(target, target_workdir)

    # Generate code, then create a synthetic target.
    task.execute_codegen(target, target_workdir)
    sources = task._capture_sources((vt,))[0]
    syn_target = task._inject_synthetic_target(vt, sources)

    actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)]
    expected_sources = syn_target.sources_relative_to_source_root()
    self.assertEqual(set(expected_sources), set(actual_sources))

    # Check that the synthetic target has a valid source root and the generated sources have the
    # expected java package
    def get_package(path):
      with open(path, 'r') as fp:
        for line in fp:
          match = self.PACKAGE_RE.match(line)
          if match:
            return match.group('package_name')
        return None

    for source in syn_target.sources_relative_to_source_root():
      source_path = os.path.join(target_workdir, source)
      self.assertTrue(os.path.isfile(source_path),
                      "{0} is not the source root for {1}".format(target_workdir, source))
      self.assertEqual(expected_package, get_package(source_path))

      self.assertIn(syn_target, context.targets())

    # Check that the output file locations match the package
    if expected_package is not None:
      expected_path_prefix = expected_package.replace('.', os.path.sep) + os.path.sep
      for source in syn_target.sources_relative_to_source_root():
        self.assertTrue(source.startswith(expected_path_prefix),
                        "{0} does not start with {1}".format(source, expected_path_prefix))

    # Check that empty directories have been removed
    for root, dirs, files in os.walk(target_workdir):
      for d in dirs:
        full_dir = os.path.join(root, d)
        self.assertTrue(os.listdir(full_dir),
                         "Empty directories should have been removed ({0})".format(full_dir))

    return syn_target

 def process_stats_file(self):
   for filename in Fileset.walk(self._stats_dir):
     try :
       with open(os.path.join(self._stats_dir, filename), 'r') as stats_file:
         lines = stats_file.readlines()
         tmp_str = ",".join(lines)
         tmp_str.strip(',')
         self.push_stats("[" + tmp_str + "]")
       os.remove(os.path.join(self._stats_dir, filename))
     except httplib.HTTPException as e:
       log.debug("HTTPException %s" % e)
     except OSError as e:
       log.debug("Error reading or deleting a stats file %s" % e)

  def test_antlr4(self):
    parts = {'srcroot': 'src/antlr',
             'dir': 'this/is/a/directory',
             'name': 'smoke',
             'package': 'this.is.a.package',
             'prefix': 'SMOKE'}
    self.create_file(relpath='%(srcroot)s/%(dir)s/%(prefix)s.g4' % parts,
                     contents=dedent('''
      grammar %(prefix)s;
      options { language=Java; }
      ////////////////////
      start  : letter EOF ;
      letter : LETTER ;
      ////////////////////
      fragment LETTER : [a-zA-Z] ;
    ''' % parts))
    self.add_to_build_file('%(srcroot)s/%(dir)s/BUILD' % parts, dedent('''
      java_antlr_library(
        name='%(name)s',
        compiler='antlr4',
        package='%(package)s',
        sources=['%(prefix)s.g4'],
      )
    ''' % parts))

    # generate a context to contain the build graph for the input target, then execute
    context = self.context(target_roots=[self.target('%(srcroot)s/%(dir)s:%(name)s' % parts)])
    task = self.execute(context, AntlrGen)

    # get the synthetic target from the private graph
    task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java')
    syn_sourceroot = os.path.join(task_outdir, parts['srcroot'])
    syn_target_name = ('%(srcroot)s/%(dir)s.%(name)s' % parts).replace('/', '.')
    syn_address = SyntheticAddress(spec_path=os.path.relpath(syn_sourceroot, self.build_root),
                                   target_name=syn_target_name)
    syn_target = context.build_graph.get_target(syn_address)

    # verify that the synthetic target's list of sources match what are actually created
    def re_relativize(p):
      """Take a path relative to task_outdir, and make it relative to the build_root"""
      # TODO: is there a way to do this directly with rglobs?
      return os.path.relpath(os.path.join(task_outdir, p), self.build_root)
    actual_sources = [re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir)]
    self.assertEquals(set(syn_target.sources_relative_to_buildroot()), set(actual_sources))
    # and that the synthetic target has a valid sourceroot
    for source in syn_target.sources_relative_to_source_root():
      self.assertTrue(os.path.isfile(os.path.join(syn_sourceroot, source)),
                      "%s is not the sourceroot for %s" % (syn_sourceroot, source))

  def __init__(self, address=None, payload=None, **kwargs):
    # TODO(John Sirois): Make pants.backend.core.wrapped_globs.Globs in the core backend
    # constructable with just a rel_path. Right now it violates the Law of Demeter and
    # fundamentally takes a ParseContext, which it doesn't actually need and which other backend
    # consumers should not need to know about or create.
    # Here we depend on twitter/commons which is less than ideal in core pants and even worse in a
    # plugin.  We depend on it here to ensure the globbing is lazy and skipped if the target is
    # never fingerprinted (eg: when running `./pants list`).
    sources = Fileset.globs('*.go', root=os.path.join(get_buildroot(), address.spec_path))

    payload = payload or Payload()
    payload.add_fields({
      'sources': self.create_sources_field(sources=sources,
                                           sources_rel_path=address.spec_path,
                                           key_arg='sources'),
    })
    super(GoLocalSource, self).__init__(address=address, payload=payload, **kwargs)

  def test_antlr_py_gen(self):
    self.create_file(
      relpath='foo/bar/baz/Baz.g',
      contents=dedent("""
        grammar Baz;

        options {
          language=Python;
          output=template;
        }

        a : ID INT
            -> template(id={$ID.text}, int={$INT.text})
               "id=<id>, int=<int>"
          ;

        ID : 'a'..'z'+;
        INT : '0'..'9'+;
        WS : (' '|'\n') {$channel=HIDDEN;} ;
      """))

    self.add_to_build_file('foo/bar/baz/BUILD', dedent("""
        python_antlr_library(
          name='baz',
          module='foo.bar.baz',
          sources=['Baz.g'],
        )
      """))

    target = self.target('foo/bar/baz')
    context = self.context(target_roots=[target])
    task = self.prepare_execute(context)
    target_workdir = self.test_workdir

    # Generate code, then create a synthetic target.
    task.execute_codegen(target, target_workdir)
    actual_sources = {s for s in Fileset.rglobs('*.py', root=target_workdir)}
    self.assertSetEqual({
      'foo/__init__.py',
      'foo/bar/__init__.py',
      'foo/bar/baz/__init__.py',
      'foo/bar/baz/BazParser.py',
      'foo/bar/baz/BazLexer.py',
    }, actual_sources)

 def zglobs_following_symlinked_dirs_by_default(*globspecs, **kw):
   if 'follow_links' not in kw:
     kw['follow_links'] = True
   return Fileset.zglobs(*globspecs, **kw)

 def rglobs_following_symlinked_dirs_by_default(*globspecs, **kw):
     if "follow_links" not in kw:
         kw["follow_links"] = True
     return Fileset.rglobs(*globspecs, **kw)