Here you have a working code.
I would suggest you to avoid the inline onclick="abc()" and opt in favor of a fully-separated code using EventListener (that's good for maintainability).
With Window.getComputedStyle() you get the background color in RGBA; you can then convert it to the HEX code with a simple function that you can find everywhere on the Web, here I used one of them. So, the right way to get the background color is window.getComputedStyle(btn, null)["backgroundColor"] while, if you would like to set it, the correct form would be document.getElementById("btn").style.backgroundColor = "#0000".
/**
 * The code inside the function is run only when the DOM is ready.
 * This is the only jQuery function used, all the rest is in vanillaJS.
 */
$(document).ready(function() {
  /**
   * rgb2hex
   * Convert RGB to HEX.
   * Source: https://jsfiddle.net/Mottie/xcqpF/1/light/
   * 
   * @param {string} rgb
   */
  var rgb2hex = function(rgb){
    rgb = rgb.match(/^rgba?[\s+]?\([\s+]?(\d+)[\s+]?,[\s+]?(\d+)[\s+]?,[\s+]?(\d+)[\s+]?/i);
    return (rgb && rgb.length === 4) ? "#" +
        ("0" + parseInt(rgb[1],10).toString(16)).slice(-2) +
        ("0" + parseInt(rgb[2],10).toString(16)).slice(-2) +
        ("0" + parseInt(rgb[3],10).toString(16)).slice(-2) : '';
  }
  /**
   * EventListener of btn click event
   */
  document.getElementById("btn")
    .addEventListener('click', function() {
      // Save the btn into a var so you can use it later
      var btn = document.getElementById("btn");
      // Notice: getComputedStyle() to get the element background color
      var color = window.getComputedStyle(btn, null)["backgroundColor"];
      // Transform RGBa to HEX
      var hex = rgb2hex(color);
      
      // IF-ELSE with ternary operators
      (hex === "#c1580b")
          ? btn.style.cssText = "box-shadow: 0px 0px 0px 3px #173B0B; background-color: #173B0B; color:#459c5c"
          : btn.style.cssText = "box-shadow: 0px 0px 0px 3px #c1580b; background-color: #c1580b; color:#ffb734";
      
    });
    
});
.btn {
  background: #c1580b;
  color: #ffb734;
  width: 80px;
  height: 80px;
  line-height: 70px;
  display: block;
  border-radius: 50%;
  text-align: center;
  text-decoration: none;
  border: 2px solid #000;
  box-shadow: 0px 0px 0px 3px #c1580b;
  font-size: medium;
  cursor: pointer;
}
<!DOCTYPE html>
<html>
  <head>
    <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
  </head>
  <body>
    <button id="btn" class="btn">Pause</button>
  </body>
</html>