graphite work log - 6/3/2015

  • monprod.example.com

  • configure collectl daemon:

mv /etc/collectl.conf /etc/collectl.conf.dist
echo 'DaemonCommands = -f /var/log/collectl -m -F60 -r00:00,90:3 -scdnmYZ' >/etc/collectl.conf
  • enable epel repo:

    sed -i '/^.epel.$/,/^$/{s/abled=0/abled=1/}' /etc/yum.repos.d/epel.repo
  • install graphite:

yum install -y python-devel Django14 django-tagging python-cairocffi
pip install whisper
pip install carbon
pip install graphite-web
  • create graphite-web config:

  cat >/opt/graphite/webapp/graphite/local_settings.py <<EOM
  SECRET_KEY = '`dd if=/dev/urandom bs=33 count=1 2>/dev/null | base64`'
  ALLOWED_HOSTS = [ '*' ]
  TIME_ZONE = 'UTC'
  DEBUG = False
  EOM
  • configure carbon:

  cd /opt/graphite/conf
  cat >carbon.conf <<EOM
  [cache]
  ENABLE_LOGROTATION = True
  USER =
  MAX_CACHE_SIZE = inf
  MAX_UPDATES_PER_SECOND = 500
  MAX_CREATES_PER_MINUTE = 50
  LINE_RECEIVER_INTERFACE = 0.0.0.0
  LINE_RECEIVER_PORT = 2003
  ENABLE_UDP_LISTENER = True
  UDP_RECEIVER_INTERFACE = 0.0.0.0
  UDP_RECEIVER_PORT = 2003
  LOG_LISTENER_CONNECTIONS = True
  USE_INSECURE_UNPICKLER = False
  CACHE_QUERY_INTERFACE = 0.0.0.0
  CACHE_QUERY_PORT = 7002
  USE_FLOW_CONTROL = True
  LOG_UPDATES = False
  LOG_CACHE_HITS = False
  LOG_CACHE_QUEUE_SORTS = True
  CACHE_WRITE_STRATEGY = naive
  WHISPER_AUTOFLUSH = False
  WHISPER_FALLOCATE_CREATE = False
  EOM

  cat >storage-schemas.conf <<EOM
  # default - 10 second resolution, save for 90 days
  [default]
  pattern = .*
  retentions = 10s:90d
  EOM

  touch storage-aggregation.conf  # stop useless log entries every min.
  • make it (a lot) easier to find carbon logs:

    ln -s /opt/graphite/storage/log/carbon-cache /var/log/carbon
  • create boot script and start carbon:

  cat > /etc/init.d/graphite <<EOM
  #!/bin/sh
  # for chkconfig, need chkconfig and description lines .
  # next line: start in runlevels: 3, start/stop priorities: 95, 65
  # chkconfig: 3 95 65
  # description: rc script for graphite
  # dcapella - 3/4/11

  bindir=/opt/graphite/bin

  case "\$1" in
    start)
      \$bindir/carbon-cache.py start
      exec \$bindir/run-graphite-devel-server.py \
        /opt/graphite </dev/null >>/var/log/graphite 2>>/var/log/graphite &
      echo \$! >/var/run/graphite.pid
      ;;
    stop)
      \$bindir/carbon-cache.py stop
      kill `cat /var/run/graphite.pid`
      ;;
    status)
      \$bindir/carbon-cache.py status
      ps -p `cat /var/run/graphite.pid`
      ;;
    *)
      echo "usage: `basename \$0` start|stop|status"
      ;;
  esac
  EOM
  chmod 755 /etc/init.d/graphite
  chkconfig --add graphite
  • work around broken django auth module:

# change line 85: return getpass.getuser().decode(locale.getdefaultlocale()[1])
# to:             return getpass.getuser()
vi /usr/lib/python2.6/site-packages/django/contrib/auth/management/__init__.py
  • create graphite-web database:

    PYTHONPATH=/opt/graphite/webapp django-admin syncdb --settings=graphite.settings
  • start carbon and web server:

service graphite start
tail /var/log/carbon/*/* /var/log/graphite
service graphite status
test:

lynx http:_monprod:8080/