hadoop-client properties

Mon Apr 11 13:30:10 UTC 2016

Hi all,

I am trying to setup a cluster with juju in the local environment to
submit jobs with Apache Giraph. You can find the details of my setup
at the end of this e-mail.

I have downloaded and build Apache Giraph on my hadoop-client and I
want to try some examples that execute on two workers.

After a number of failed attempts I found out that I have to set
property: ''mapreduce.jobtracker.address' (or the deprecated
'mapred.job.tracker') to 'yarn' in order to run giraph with > 1
workers.

In particular, Giraph considered that this property was set to 'local'.
At first I found out that I can set a custom attribute with:
-ca giraph.SplitMasterWorker=false
to execute my job with one worker.
Then, after finding the code responsible for this
behavior(https://github.com/apache/giraph/blob/7e48523b520afee8e727d1e1aaab801a3bd80f06/giraph-core/src/main/java/org/apache/giraph/job/GiraphJob.java#L143)
I was able to set the correct hadoop property and execute my job with
2 workers.

My question is, why is this property not set in the juju client charm?
Does it enable some otherwise undesired behavior?
I see that 'mapreduce.framework.name' is set to 'yarn' but apparently
this is not enough for giraph.

Thank you.

--Panagiotis Liakos

My environment:

environment: local
machines:
  "0":
    agent-state: started
    agent-version: 1.25.3.1
    dns-name: localhost
    instance-id: localhost
    series: trusty
    state-server-member-status: has-vote
  "1":
    agent-state: started
    agent-version: 1.25.3.1
    dns-name: 10.0.3.78
    instance-id: panagiotis-local-machine-1
    series: trusty
    hardware: arch=amd64
  "2":
    agent-state: started
    agent-version: 1.25.3.1
    dns-name: 10.0.3.178
    instance-id: panagiotis-local-machine-2
    series: trusty
    hardware: arch=amd64
  "3":
    agent-state: started
    agent-version: 1.25.3.1
    dns-name: 10.0.3.182
    instance-id: panagiotis-local-machine-3
    series: trusty
    hardware: arch=amd64
  "4":
    agent-state: started
    agent-version: 1.25.3.1
    dns-name: 10.0.3.179
    instance-id: panagiotis-local-machine-4
    series: trusty
    hardware: arch=amd64
  "5":
    agent-state: started
    agent-version: 1.25.3.1
    dns-name: 10.0.3.183
    instance-id: panagiotis-local-machine-5
    series: trusty
    hardware: arch=amd64
services:
  client:
    charm: cs:trusty/hadoop-client-1
    exposed: false
    service-status:
      current: active
      message: Ready
      since: 10 Apr 2016 08:08:20+03:00
    relations:
      hadoop:
      - plugin
    units:
      client/0:
        workload-status:
          current: active
          message: Ready
          since: 10 Apr 2016 08:08:20+03:00
        agent-status:
          current: idle
          since: 10 Apr 2016 08:08:23+03:00
          version: 1.25.3.1
        agent-state: started
        agent-version: 1.25.3.1
        machine: "5"
        public-address: 10.0.3.183
        subordinates:
          plugin/0:
            workload-status:
              current: active
              message: Ready (HDFS & YARN)
              since: 10 Apr 2016 08:08:01+03:00
            agent-status:
              current: idle
              since: 10 Apr 2016 08:08:03+03:00
              version: 1.25.3.1
            agent-state: started
            agent-version: 1.25.3.1
            upgrading-from: cs:trusty/apache-hadoop-plugin-13
            public-address: 10.0.3.183
  namenode:
    charm: cs:trusty/apache-hadoop-namenode-1
    exposed: false
    service-status:
      current: active
      message: Ready (2 DataNodes)
      since: 10 Apr 2016 08:04:33+03:00
    relations:
      datanode:
      - slave
      namenode:
      - plugin
      - resourcemgr
    units:
      namenode/0:
        workload-status:
          current: active
          message: Ready (2 DataNodes)
          since: 10 Apr 2016 08:04:33+03:00
        agent-status:
          current: idle
          since: 10 Apr 2016 08:04:36+03:00
          version: 1.25.3.1
        agent-state: started
        agent-version: 1.25.3.1
        machine: "1"
        open-ports:
        - 50070/tcp
        public-address: 10.0.3.78
  plugin:
    charm: cs:trusty/apache-hadoop-plugin-13
    exposed: false
    service-status: {}
    relations:
      hadoop-plugin:
      - client
      namenode:
      - namenode
      resourcemanager:
      - resourcemgr
    subordinate-to:
    - client
  resourcemgr:
    charm: cs:trusty/apache-hadoop-resourcemanager-1
    exposed: false
    service-status:
      current: active
      message: Ready (2 NodeManagers)
      since: 10 Apr 2016 08:07:54+03:00
    relations:
      namenode:
      - namenode
      nodemanager:
      - slave
      resourcemanager:
      - plugin
    units:
      resourcemgr/0:
        workload-status:
          current: active
          message: Ready (2 NodeManagers)
          since: 10 Apr 2016 08:07:54+03:00
        agent-status:
          current: idle
          since: 10 Apr 2016 08:07:57+03:00
          version: 1.25.3.1
        agent-state: started
        agent-version: 1.25.3.1
        machine: "2"
        open-ports:
        - 8088/tcp
        - 19888/tcp
        public-address: 10.0.3.178
  slave:
    charm: cs:trusty/apache-hadoop-slave-1
    exposed: false
    service-status:
      current: active
      message: Ready (DataNode & NodeManager)
      since: 10 Apr 2016 08:08:00+03:00
    relations:
      namenode:
      - namenode
      resourcemanager:
      - resourcemgr
    units:
      slave/0:
        workload-status:
          current: active
          message: Ready (DataNode & NodeManager)
          since: 10 Apr 2016 08:08:00+03:00
        agent-status:
          current: idle
          since: 10 Apr 2016 08:08:03+03:00
          version: 1.25.3.1
        agent-state: started
        agent-version: 1.25.3.1
        machine: "3"
        open-ports:
        - 8042/tcp
        - 50075/tcp
        public-address: 10.0.3.182
      slave/1:
        workload-status:
          current: active
          message: Ready (DataNode & NodeManager)
          since: 10 Apr 2016 08:08:30+03:00
        agent-status:
          current: idle
          since: 10 Apr 2016 08:08:33+03:00
          version: 1.25.3.1
        agent-state: started
        agent-version: 1.25.3.1
        machine: "4"
        open-ports:
        - 8042/tcp
        - 50075/tcp
        public-address: 10.0.3.179