Answer:
When you check if the element exists you are actually looking at an Array. To determine if the Array is not empty, and therefore, the class exists within the category element, you just need to test the length property. 
If the Array length is 0 and nothing is in it, it will return false. If the Array length is greater than 0, something is in it, and it will return true.
Secondly when you utilize properties that have a hyphen( - ) you need to pass that property as a String within the object you're passing to the css method of JQuery's element wrapper. 
var elementExists = document.getElementById('category')
                    .getElementsByClassName('st_banner_row')
                    .length;
if (elementExists)
{
     $("#search_filters_wrapper").css({"margin-top": 40});
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<div id='category'>
<span>Below is 40px</span>
     <div id="search_filters_wrapper">
      Filter
      </div>  
      <div class="st_banner_row" style="">
       There is Banner
      </div>
</div>
 
 
Aside:
It's odd that you're using JQuery for this one aspect of code. It would be easier and more maintainable to use either all Vanilla JavaScript or all JQuery.
JQuery:
var elementExists = $("#category").find(".st_banner_row").length;
if (elementExists)
{
     $("#search_filters_wrapper").css({"margin-top": 40});
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<div id='category'>
<span>Below is 40px</span>
     <div id="search_filters_wrapper">
      Filter
      </div>  
      <div class="st_banner_row" style="">
       There is Banner
      </div>
</div>
 
 
JavaScript:
var elementExists = document.getElementById('category')
                    .getElementsByClassName('st_banner_row')
                    .length;
if (elementExists)
{
     document.getElementById("search_filters_wrapper").style.marginTop = "40px";
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<div id='category'>
<span>Below is 40px</span>
     <div id="search_filters_wrapper">
      Filter
      </div>  
      <div class="st_banner_row" style="">
       There is Banner
      </div>
</div>
 
 
Alternative Vanilla JS using querySelector:
var elementExists = document.querySelector('#category .st_banner_row');
if (elementExists)
{
     document.querySelector("#search_filters_wrapper").style.marginTop = "40px";
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<div id='category'>
<span>Below is 40px</span>
     <div id="search_filters_wrapper">
      Filter
      </div>  
      <div class="st_banner_row" style="">
       There is Banner
      </div>
</div>
 
 
Note In this version we don't need to check the length because if category does not have a child with a class of st_banner_row the selector will return undefined. 
Alternative Vanilla JavaScript Functional example:
// helper functions
const el = ( query, context = document ) => context.querySelector( query ),
 elementExists = query => Boolean( el( query ) ),
 ifElementExists = ( query, fn = () => undefined ) => elementExists( query ) && fn(),
 elStyle = query => ( prop, value ) => el( query ).style[ prop ] = value,
 changeStyle = query => ( prop, value ) => () => elStyle( query )( prop, value );
// execution
ifElementExists( "#category .st_banner_row", 
    changeStyle( "#search_filters_wrapper" )( "margin-top", "40px" ) 
);
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<div id='category'>
  <span>Below is 40px</span>
  <div id="search_filters_wrapper">
    Filter
  </div>
  <div class="st_banner_row" style="">
    There is Banner
  </div>
</div>