One way could be to convert the points and polygon to sf objects, crop the points, then turn the data back into a tibble.
library(sf)
library(tidyverse)
library(gridExtra)
set.seed(345)
tbl <- tibble(x = runif(100),
              y = runif(100))
poly <- tibble(
  A = c(0.5, 0),
  B = c(1, 0.5),
  C = c(0.5, 1),
  D = c(0, 0.5)
)
# Convert tbl to sf object.
tbl_sf <- sf::st_as_sf(tbl, coords = c("x", "y"))
# Convert polygon to sf polygon.
poly_sf <- as.data.frame(t(poly)) %>%
  dplyr::rename(x = V1, y = V2) %>%
  sf::st_as_sf(coords = c("x", "y")) %>%
  dplyr::summarise() %>%
  sf::st_cast("POLYGON") %>%
  sf::st_convex_hull()
# Keep only points inside the polygon.
points <- sf::st_intersection(tbl_sf, poly_sf) %>%
  
# Get coordinates and convert back to tibble.
  sf::st_coordinates() %>%
  as_tibble() %>%
  dplyr::rename(x = X, y = Y)
before <- ggplot(data = tbl,
                 aes(x = x,
                     y = y)) +
  geom_point() +
  theme_bw() +
  coord_equal() +
  ggtitle("Before")
after <- ggplot(data = points,
                aes(x = x,
                    y = y)) +
  geom_point() +
  theme_bw() +
  coord_equal() +
  ggtitle("After")
grid.arrange(before, after, ncol = 2)
Output
